nicolas/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Correct ordering of loads/stores.

Browse Source 
Summary:
Aiming to correct the ordering of loads/stores. This patch changes the
insert point for loads to the position of the first load.
It updates the ordering method for loads to insert before, rather than after.

Before this patch the following sequence:
"load a[1], store a[1], store a[0], load a[2]"
Would incorrectly vectorize to "store a[0,1], load a[1,2]".
The correctness check was assuming the insertion point for loads is at
the position of the first load, when in practice it was at the last
load. An alternative fix would have been to invert the correctness check.
The current fix changes insert position but also requires reordering of
instructions before the vectorized load.

Updated testcases to reflect the changes.

Reviewers: tstellarAMD, llvm-commits, jlebar, arsenm

Subscribers: mzolotukhin

Differential Revision: http://reviews.llvm.org/D22071

llvm-svn: 275117
This commit is contained in:
Alina Sbirlea
2016-07-11 22:34:29 +00:00
parent 3e0361710a
commit cbc6ac2afd
6 changed files with 212 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
View File

@@ -88,8 +88,8 @@ private:
bool isConsecutiveAccess(Value *A, Value *B);
/// Reorders the users of I after vectorization to ensure that I dominates its
/// users.
/// After vectorization, reorder the instructions that I depends on
/// (the instructions defining its operands), to ensure they dominate I.
void reorder(Instruction *I);
/// Returns the first and the last instructions in Chain.
@@ -334,19 +334,37 @@ bool Vectorizer::isConsecutiveAccess(Value *A, Value *B) {
}
void Vectorizer::reorder(Instruction *I) {
Instruction *InsertAfter = I;
for (User *U : I->users()) {
Instruction *User = dyn_cast<Instruction>(U);
if (!User || User->getOpcode() == Instruction::PHI)
continue;
SmallPtrSet<Instruction *, 16> InstructionsToMove;
SmallVector<Instruction *, 16> Worklist;
if (!DT.dominates(I, User)) {
User->removeFromParent();
User->insertAfter(InsertAfter);
InsertAfter = User;
reorder(User);
Worklist.push_back(I);
while (!Worklist.empty()) {
Instruction *IW = Worklist.pop_back_val();
int NumOperands = IW->getNumOperands();
for (int i = 0; i < NumOperands; i++) {
Instruction *IM = dyn_cast<Instruction>(IW->getOperand(i));
if (!IM || IM->getOpcode() == Instruction::PHI)
continue;
if (!DT.dominates(IM, I)) {
InstructionsToMove.insert(IM);
Worklist.push_back(IM);
assert(IM->getParent() == IW->getParent() &&
"Instructions to move should be in the same basic block");
}
}
}
// All instructions to move should follow I. Start from I, not from begin().
for (auto BBI = I->getIterator(), E = I->getParent()->end(); BBI != E;
++BBI) {
if (!is_contained(InstructionsToMove, &*BBI))
continue;
Instruction *IM = &*BBI;
--BBI;
IM->removeFromParent();
IM->insertBefore(I);
}
}
std::pair<BasicBlock::iterator, BasicBlock::iterator>
@@ -851,7 +869,7 @@ bool Vectorizer::vectorizeLoadChain(ArrayRef<Value *> Chain) {
return false;
// Set insert point.
Builder.SetInsertPoint(&*Last);
Builder.SetInsertPoint(&*First);
Value *Bitcast =
Builder.CreateBitCast(L0->getPointerOperand(), VecTy->getPointerTo(AS));
@@ -863,6 +881,7 @@ bool Vectorizer::vectorizeLoadChain(ArrayRef<Value *> Chain) {
if (VecLoadTy) {
SmallVector<Instruction *, 16> InstrsToErase;
SmallVector<Instruction *, 16> InstrsToReorder;
InstrsToReorder.push_back(cast<Instruction>(Bitcast));
unsigned VecWidth = VecLoadTy->getNumElements();
for (unsigned I = 0, E = Chain.size(); I != E; ++I) {
@@ -878,7 +897,6 @@ bool Vectorizer::vectorizeLoadChain(ArrayRef<Value *> Chain) {
// Replace the old instruction.
UI->replaceAllUsesWith(Extracted);
InstrsToReorder.push_back(Extracted);
InstrsToErase.push_back(UI);
}
}
@@ -890,6 +908,7 @@ bool Vectorizer::vectorizeLoadChain(ArrayRef<Value *> Chain) {
I->eraseFromParent();
} else {
SmallVector<Instruction *, 16> InstrsToReorder;
InstrsToReorder.push_back(cast<Instruction>(Bitcast));
for (unsigned I = 0, E = Chain.size(); I != E; ++I) {
Value *V = Builder.CreateExtractElement(LI, Builder.getInt32(I));
@@ -902,7 +921,6 @@ bool Vectorizer::vectorizeLoadChain(ArrayRef<Value *> Chain) {
// Replace the old instruction.
UI->replaceAllUsesWith(Extracted);
InstrsToReorder.push_back(Extracted);
}
for (Instruction *ModUser : InstrsToReorder)