From e997903e5d29adb83d8dfa435ff05f3cc87d4ee7 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 29 Aug 2010 08:53:19 +0000 Subject: [PATCH] 2010-08-29 Tatsuhiro Tsujikawa Added a sets of overloaded functions for PieceStorage::getMissingPiece() and getMissingFastPiece() to get multiple pieces more efficiently. * src/DefaultBtInteractive.cc * src/DefaultPieceStorage.cc * src/DefaultPieceStorage.h * src/PieceStorage.h * src/UnknownLengthPieceStorage.cc * src/UnknownLengthPieceStorage.h * test/DefaultPieceStorageTest.cc * test/MockPieceStorage.h --- ChangeLog | 14 +++ src/DefaultBtInteractive.cc | 55 +++++---- src/DefaultPieceStorage.cc | 201 +++++++++++++++++++++---------- src/DefaultPieceStorage.h | 36 ++++-- src/PieceStorage.h | 22 ++++ src/UnknownLengthPieceStorage.cc | 34 ++++++ src/UnknownLengthPieceStorage.h | 22 ++++ test/DefaultPieceStorageTest.cc | 45 ++++++- test/MockPieceStorage.h | 26 ++++ 9 files changed, 351 insertions(+), 104 deletions(-) diff --git a/ChangeLog b/ChangeLog index 11d5547c..1585c29f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,17 @@ +2010-08-29 Tatsuhiro Tsujikawa + + Added a sets of overloaded functions for + PieceStorage::getMissingPiece() and getMissingFastPiece() to get + multiple pieces more efficiently. + * src/DefaultBtInteractive.cc + * src/DefaultPieceStorage.cc + * src/DefaultPieceStorage.h + * src/PieceStorage.h + * src/UnknownLengthPieceStorage.cc + * src/UnknownLengthPieceStorage.h + * test/DefaultPieceStorageTest.cc + * test/MockPieceStorage.h + 2010-08-28 Tatsuhiro Tsujikawa Changed signature of util::executeHook(). diff --git a/src/DefaultBtInteractive.cc b/src/DefaultBtInteractive.cc index a9ed1dbf..a871b532 100644 --- a/src/DefaultBtInteractive.cc +++ b/src/DefaultBtInteractive.cc @@ -355,42 +355,41 @@ void DefaultBtInteractive::decideInterest() { void DefaultBtInteractive::fillPiece(size_t maxMissingBlock) { if(pieceStorage_->hasMissingPiece(peer_)) { - size_t numMissingBlock = btRequestFactory_->countMissingBlock(); - + if(numMissingBlock >= maxMissingBlock) { + return; + } + size_t diffMissingBlock = maxMissingBlock-numMissingBlock; + std::vector > pieces; if(peer_->peerChoking()) { if(peer_->isFastExtensionEnabled()) { - std::vector excludedIndexes; - excludedIndexes.reserve(btRequestFactory_->countTargetPiece()); - btRequestFactory_->getTargetPieceIndexes(excludedIndexes); - while(numMissingBlock < maxMissingBlock) { - SharedHandle piece = - pieceStorage_->getMissingFastPiece(peer_, excludedIndexes); - if(piece.isNull()) { - break; - } else { - btRequestFactory_->addTargetPiece(piece); - numMissingBlock += piece->countMissingBlock(); - excludedIndexes.push_back(piece->getIndex()); - } + if(pieceStorage_->isEndGame()) { + std::vector excludedIndexes; + excludedIndexes.reserve(btRequestFactory_->countTargetPiece()); + btRequestFactory_->getTargetPieceIndexes(excludedIndexes); + pieceStorage_->getMissingFastPiece + (pieces, diffMissingBlock, peer_, excludedIndexes); + } else { + pieces.reserve(diffMissingBlock); + pieceStorage_->getMissingFastPiece(pieces, diffMissingBlock, peer_); } } } else { - std::vector excludedIndexes; - excludedIndexes.reserve(btRequestFactory_->countTargetPiece()); - btRequestFactory_->getTargetPieceIndexes(excludedIndexes); - while(numMissingBlock < maxMissingBlock) { - SharedHandle piece = - pieceStorage_->getMissingPiece(peer_, excludedIndexes); - if(piece.isNull()) { - break; - } else { - btRequestFactory_->addTargetPiece(piece); - numMissingBlock += piece->countMissingBlock(); - excludedIndexes.push_back(piece->getIndex()); - } + if(pieceStorage_->isEndGame()) { + std::vector excludedIndexes; + excludedIndexes.reserve(btRequestFactory_->countTargetPiece()); + btRequestFactory_->getTargetPieceIndexes(excludedIndexes); + pieceStorage_->getMissingPiece + (pieces, diffMissingBlock, peer_, excludedIndexes); + } else { + pieces.reserve(diffMissingBlock); + pieceStorage_->getMissingPiece(pieces, diffMissingBlock, peer_); } } + for(std::vector >::const_iterator i = + pieces.begin(), eoi = pieces.end(); i != eoi; ++i) { + btRequestFactory_->addTargetPiece(*i); + } } } diff --git a/src/DefaultPieceStorage.cc b/src/DefaultPieceStorage.cc index bf1f0a77..cc90fb7d 100644 --- a/src/DefaultPieceStorage.cc +++ b/src/DefaultPieceStorage.cc @@ -59,6 +59,7 @@ #include "array_fun.h" #include "PieceStatMan.h" #include "wallclock.h" +#include "bitfield.h" #ifdef ENABLE_BITTORRENT # include "bittorrent_helper.h" #endif // ENABLE_BITTORRENT @@ -83,29 +84,6 @@ DefaultPieceStorage::~DefaultPieceStorage() { delete bitfieldMan_; } -bool DefaultPieceStorage::getMissingPieceIndex(size_t& index, - const unsigned char* bitfield, - size_t length) -{ - const size_t mislen = bitfieldMan_->getBitfieldLength(); - array_ptr misbitfield(new unsigned char[mislen]); - bool r; - if(isEndGame()) { - r = bitfieldMan_->getAllMissingIndexes(misbitfield, mislen, - bitfield, length); - } else { - r = bitfieldMan_->getAllMissingUnusedIndexes(misbitfield, mislen, - bitfield, length); - } - if(r) { - // We assume indexes is sorted using comparator less. - return - pieceSelector_->select(index, misbitfield,bitfieldMan_->countBlock()); - } else { - return false; - } -} - SharedHandle DefaultPieceStorage::checkOutPiece(size_t index) { bitfieldMan_->setUseBit(index); @@ -172,23 +150,6 @@ SharedHandle DefaultPieceStorage::findUsedPiece(size_t index) const } } -SharedHandle DefaultPieceStorage::getMissingPiece -(const unsigned char* bitfield, size_t length) -{ - size_t index; - if(getMissingPieceIndex(index, bitfield, length)) { - return checkOutPiece(index); - } else { - return SharedHandle(); - } -} - -SharedHandle DefaultPieceStorage::getMissingPiece -(const BitfieldMan& bitfield) -{ - return getMissingPiece(bitfield.getBitfield(), bitfield.getBitfieldLength()); -} - #ifdef ENABLE_BITTORRENT bool DefaultPieceStorage::hasMissingPiece(const SharedHandle& peer) @@ -197,10 +158,58 @@ bool DefaultPieceStorage::hasMissingPiece(const SharedHandle& peer) peer->getBitfieldLength()); } -SharedHandle -DefaultPieceStorage::getMissingPiece(const SharedHandle& peer) +void DefaultPieceStorage::getMissingPiece +(std::vector >& pieces, + size_t minMissingBlocks, + const unsigned char* bitfield, + size_t length) { - return getMissingPiece(peer->getBitfield(), peer->getBitfieldLength()); + const size_t mislen = bitfieldMan_->getBitfieldLength(); + array_ptr misbitfield(new unsigned char[mislen]); + size_t blocks = bitfieldMan_->countBlock(); + size_t misBlock = 0; + if(isEndGame()) { + bool r = bitfieldMan_->getAllMissingIndexes + (misbitfield, mislen, bitfield, length); + if(!r) { + return; + } + std::vector indexes; + for(size_t i = 0; i < blocks; ++i) { + if(bitfield::test(misbitfield, blocks, i)) { + indexes.push_back(i); + } + } + std::random_shuffle(indexes.begin(), indexes.end()); + for(std::vector::const_iterator i = indexes.begin(), + eoi = indexes.end(); i != eoi && misBlock < minMissingBlocks; ++i) { + pieces.push_back(checkOutPiece(*i)); + misBlock += pieces.back()->countMissingBlock(); + } + } else { + bool r = bitfieldMan_->getAllMissingUnusedIndexes + (misbitfield, mislen, bitfield, length); + if(!r) { + return; + } + while(misBlock < minMissingBlocks) { + size_t index; + if(pieceSelector_->select(index, misbitfield, blocks)) { + pieces.push_back(checkOutPiece(index)); + bitfield::flipBit(misbitfield, blocks, index); + misBlock += pieces.back()->countMissingBlock(); + } else { + break; + } + } + } +} + +static void unsetExcludedIndexes(BitfieldMan& bitfield, + const std::vector& excludedIndexes) +{ + std::for_each(excludedIndexes.begin(), excludedIndexes.end(), + std::bind1st(std::mem_fun(&BitfieldMan::unsetBit), &bitfield)); } void DefaultPieceStorage::createFastIndexBitfield @@ -215,47 +224,107 @@ void DefaultPieceStorage::createFastIndexBitfield } } -SharedHandle DefaultPieceStorage::getMissingFastPiece -(const SharedHandle& peer) +void DefaultPieceStorage::getMissingPiece +(std::vector >& pieces, + size_t minMissingBlocks, + const SharedHandle& peer) { - if(peer->isFastExtensionEnabled() && peer->countPeerAllowedIndexSet() > 0) { - BitfieldMan tempBitfield(bitfieldMan_->getBlockLength(), - bitfieldMan_->getTotalLength()); - createFastIndexBitfield(tempBitfield, peer); - return getMissingPiece(tempBitfield); - } else { - return SharedHandle(); - } + getMissingPiece(pieces, minMissingBlocks, + peer->getBitfield(), peer->getBitfieldLength()); } -static void unsetExcludedIndexes(BitfieldMan& bitfield, - const std::vector& excludedIndexes) -{ - std::for_each(excludedIndexes.begin(), excludedIndexes.end(), - std::bind1st(std::mem_fun(&BitfieldMan::unsetBit), &bitfield)); -} -SharedHandle DefaultPieceStorage::getMissingPiece -(const SharedHandle& peer, const std::vector& excludedIndexes) +void DefaultPieceStorage::getMissingPiece +(std::vector >& pieces, + size_t minMissingBlocks, + const SharedHandle& peer, + const std::vector& excludedIndexes) { BitfieldMan tempBitfield(bitfieldMan_->getBlockLength(), bitfieldMan_->getTotalLength()); tempBitfield.setBitfield(peer->getBitfield(), peer->getBitfieldLength()); unsetExcludedIndexes(tempBitfield, excludedIndexes); - return getMissingPiece(tempBitfield); + getMissingPiece(pieces, minMissingBlocks, + tempBitfield.getBitfield(), tempBitfield.getBitfieldLength()); } -SharedHandle DefaultPieceStorage::getMissingFastPiece -(const SharedHandle& peer, const std::vector& excludedIndexes) +void DefaultPieceStorage::getMissingFastPiece +(std::vector >& pieces, + size_t minMissingBlocks, + const SharedHandle& peer) +{ + if(peer->isFastExtensionEnabled() && peer->countPeerAllowedIndexSet() > 0) { + BitfieldMan tempBitfield(bitfieldMan_->getBlockLength(), + bitfieldMan_->getTotalLength()); + createFastIndexBitfield(tempBitfield, peer); + getMissingPiece(pieces, minMissingBlocks, + tempBitfield.getBitfield(), + tempBitfield.getBitfieldLength()); + } +} + +void DefaultPieceStorage::getMissingFastPiece +(std::vector >& pieces, + size_t minMissingBlocks, + const SharedHandle& peer, + const std::vector& excludedIndexes) { if(peer->isFastExtensionEnabled() && peer->countPeerAllowedIndexSet() > 0) { BitfieldMan tempBitfield(bitfieldMan_->getBlockLength(), bitfieldMan_->getTotalLength()); createFastIndexBitfield(tempBitfield, peer); unsetExcludedIndexes(tempBitfield, excludedIndexes); - return getMissingPiece(tempBitfield); - } else { + getMissingPiece(pieces, minMissingBlocks, + tempBitfield.getBitfield(), + tempBitfield.getBitfieldLength()); + } +} + +SharedHandle +DefaultPieceStorage::getMissingPiece(const SharedHandle& peer) +{ + std::vector > pieces; + getMissingPiece(pieces, 1, peer); + if(pieces.empty()) { return SharedHandle(); + } else { + return pieces.front(); + } +} + +SharedHandle DefaultPieceStorage::getMissingPiece +(const SharedHandle& peer, const std::vector& excludedIndexes) +{ + std::vector > pieces; + getMissingPiece(pieces, 1, peer, excludedIndexes); + if(pieces.empty()) { + return SharedHandle(); + } else { + return pieces.front(); + } +} + +SharedHandle DefaultPieceStorage::getMissingFastPiece +(const SharedHandle& peer) +{ + std::vector > pieces; + getMissingFastPiece(pieces, 1, peer); + if(pieces.empty()) { + return SharedHandle(); + } else { + return pieces.front(); + } +} + +SharedHandle DefaultPieceStorage::getMissingFastPiece +(const SharedHandle& peer, const std::vector& excludedIndexes) +{ + std::vector > pieces; + getMissingFastPiece(pieces, 1, peer, excludedIndexes); + if(pieces.empty()) { + return SharedHandle(); + } else { + return pieces.front(); } } diff --git a/src/DefaultPieceStorage.h b/src/DefaultPieceStorage.h index 3846d1a2..6d31015d 100644 --- a/src/DefaultPieceStorage.h +++ b/src/DefaultPieceStorage.h @@ -88,15 +88,13 @@ private: SharedHandle pieceSelector_; - bool getMissingPieceIndex(size_t& index, - const unsigned char* bitfield, size_t length); - - SharedHandle getMissingPiece(const unsigned char* bitfield, - size_t length); - - SharedHandle getMissingPiece(const BitfieldMan& bitfield); - #ifdef ENABLE_BITTORRENT + void getMissingPiece + (std::vector >& pieces, + size_t minMissingBlocks, + const unsigned char* bitfield, + size_t length); + void createFastIndexBitfield(BitfieldMan& bitfield, const SharedHandle& peer); #endif // ENABLE_BITTORRENT @@ -123,6 +121,28 @@ public: virtual bool hasMissingPiece(const SharedHandle& peer); + virtual void getMissingPiece + (std::vector >& pieces, + size_t minMissingBlocks, + const SharedHandle& peer, + const std::vector& excludedIndexes); + + virtual void getMissingPiece + (std::vector >& pieces, + size_t minMissingBlocks, + const SharedHandle& peer); + + virtual void getMissingFastPiece + (std::vector >& pieces, + size_t minMissingBlocks, + const SharedHandle& peer, + const std::vector& excludedIndexes); + + virtual void getMissingFastPiece + (std::vector >& pieces, + size_t minMissingBlocks, + const SharedHandle& peer); + virtual SharedHandle getMissingPiece(const SharedHandle& peer); virtual SharedHandle getMissingFastPiece diff --git a/src/PieceStorage.h b/src/PieceStorage.h index efaf01e2..46043367 100644 --- a/src/PieceStorage.h +++ b/src/PieceStorage.h @@ -63,6 +63,28 @@ public: */ virtual bool hasMissingPiece(const SharedHandle& peer) = 0; + virtual void getMissingPiece + (std::vector >& pieces, + size_t minMissingBlocks, + const SharedHandle& peer, + const std::vector& excludedIndexes) = 0; + + virtual void getMissingPiece + (std::vector >& pieces, + size_t minMissingBlocks, + const SharedHandle& peer) = 0; + + virtual void getMissingFastPiece + (std::vector >& pieces, + size_t minMissingBlocks, + const SharedHandle& peer, + const std::vector& excludedIndexes) = 0; + + virtual void getMissingFastPiece + (std::vector >& pieces, + size_t minMissingBlocks, + const SharedHandle& peer) = 0; + /** * Returns a piece that the peer has but localhost doesn't. * The piece will be marked "used" status in order to prevent other command diff --git a/src/UnknownLengthPieceStorage.cc b/src/UnknownLengthPieceStorage.cc index 456772b6..1bbec794 100644 --- a/src/UnknownLengthPieceStorage.cc +++ b/src/UnknownLengthPieceStorage.cc @@ -78,6 +78,40 @@ bool UnknownLengthPieceStorage::hasMissingPiece(const SharedHandle& peer) abort(); } +void UnknownLengthPieceStorage::getMissingPiece +(std::vector >& pieces, + size_t minMissingBlocks, + const SharedHandle& peer, + const std::vector& excludedIndexes) +{ + abort(); +} + +void UnknownLengthPieceStorage::getMissingPiece +(std::vector >& pieces, + size_t minMissingBlocks, + const SharedHandle& peer) +{ + abort(); +} + +void UnknownLengthPieceStorage::getMissingFastPiece +(std::vector >& pieces, + size_t minMissingBlocks, + const SharedHandle& peer, + const std::vector& excludedIndexes) +{ + abort(); +} + +void UnknownLengthPieceStorage::getMissingFastPiece +(std::vector >& pieces, + size_t minMissingBlocks, + const SharedHandle& peer) +{ + abort(); +} + SharedHandle UnknownLengthPieceStorage::getMissingPiece(const SharedHandle& peer) { abort(); diff --git a/src/UnknownLengthPieceStorage.h b/src/UnknownLengthPieceStorage.h index 5b89391a..648b944c 100644 --- a/src/UnknownLengthPieceStorage.h +++ b/src/UnknownLengthPieceStorage.h @@ -73,6 +73,28 @@ public: */ virtual bool hasMissingPiece(const SharedHandle& peer); + virtual void getMissingPiece + (std::vector >& pieces, + size_t minMissingBlocks, + const SharedHandle& peer, + const std::vector& excludedIndexes); + + virtual void getMissingPiece + (std::vector >& pieces, + size_t minMissingBlocks, + const SharedHandle& peer); + + virtual void getMissingFastPiece + (std::vector >& pieces, + size_t minMissingBlocks, + const SharedHandle& peer, + const std::vector& excludedIndexes); + + virtual void getMissingFastPiece + (std::vector >& pieces, + size_t minMissingBlocks, + const SharedHandle& peer); + /** * Returns a piece that the peer has but localhost doesn't. * The piece will be marked "used" status in order to prevent other command diff --git a/test/DefaultPieceStorageTest.cc b/test/DefaultPieceStorageTest.cc index f97847d8..f173be87 100644 --- a/test/DefaultPieceStorageTest.cc +++ b/test/DefaultPieceStorageTest.cc @@ -12,6 +12,9 @@ #include "InOrderPieceSelector.h" #include "DownloadContext.h" #include "bittorrent_helper.h" +#include "DiskAdaptor.h" +#include "DiskWriterFactory.h" +#include "PieceStatMan.h" namespace aria2 { @@ -20,7 +23,9 @@ class DefaultPieceStorageTest:public CppUnit::TestFixture { CPPUNIT_TEST_SUITE(DefaultPieceStorageTest); CPPUNIT_TEST(testGetTotalLength); CPPUNIT_TEST(testGetMissingPiece); + CPPUNIT_TEST(testGetMissingPiece_many); CPPUNIT_TEST(testGetMissingPiece_excludedIndexes); + CPPUNIT_TEST(testGetMissingPiece_manyWithExcludedIndexes); CPPUNIT_TEST(testGetMissingFastPiece); CPPUNIT_TEST(testGetMissingFastPiece_excludedIndexes); CPPUNIT_TEST(testHasMissingPiece); @@ -57,7 +62,9 @@ public: void testGetTotalLength(); void testGetMissingPiece(); + void testGetMissingPiece_many(); void testGetMissingPiece_excludedIndexes(); + void testGetMissingPiece_manyWithExcludedIndexes(); void testGetMissingFastPiece(); void testGetMissingFastPiece_excludedIndexes(); void testHasMissingPiece(); @@ -83,8 +90,6 @@ void DefaultPieceStorageTest::testGetTotalLength() { void DefaultPieceStorageTest::testGetMissingPiece() { DefaultPieceStorage pss(dctx_, option); pss.setPieceSelector(pieceSelector_); - pss.setEndGamePieceNum(0); - peer->setAllBitfield(); SharedHandle piece = pss.getMissingPiece(peer); @@ -100,6 +105,24 @@ void DefaultPieceStorageTest::testGetMissingPiece() { CPPUNIT_ASSERT(piece.isNull()); } +void DefaultPieceStorageTest::testGetMissingPiece_many() { + DefaultPieceStorage pss(dctx_, option); + pss.setPieceSelector(pieceSelector_); + peer->setAllBitfield(); + std::vector > pieces; + pss.getMissingPiece(pieces, 2, peer); + CPPUNIT_ASSERT_EQUAL((size_t)2, pieces.size()); + CPPUNIT_ASSERT_EQUAL(std::string("piece: index=0, length=128"), + pieces[0]->toString()); + CPPUNIT_ASSERT_EQUAL(std::string("piece: index=1, length=128"), + pieces[1]->toString()); + pieces.clear(); + pss.getMissingPiece(pieces, 2, peer); + CPPUNIT_ASSERT_EQUAL((size_t)1, pieces.size()); + CPPUNIT_ASSERT_EQUAL(std::string("piece: index=2, length=128"), + pieces[0]->toString()); +} + void DefaultPieceStorageTest::testGetMissingPiece_excludedIndexes() { DefaultPieceStorage pss(dctx_, option); @@ -123,6 +146,24 @@ void DefaultPieceStorageTest::testGetMissingPiece_excludedIndexes() CPPUNIT_ASSERT(piece.isNull()); } +void DefaultPieceStorageTest::testGetMissingPiece_manyWithExcludedIndexes() { + DefaultPieceStorage pss(dctx_, option); + pss.setPieceSelector(pieceSelector_); + peer->setAllBitfield(); + std::vector excludedIndexes; + excludedIndexes.push_back(1); + std::vector > pieces; + pss.getMissingPiece(pieces, 2, peer, excludedIndexes); + CPPUNIT_ASSERT_EQUAL((size_t)2, pieces.size()); + CPPUNIT_ASSERT_EQUAL(std::string("piece: index=0, length=128"), + pieces[0]->toString()); + CPPUNIT_ASSERT_EQUAL(std::string("piece: index=2, length=128"), + pieces[1]->toString()); + pieces.clear(); + pss.getMissingPiece(pieces, 2, peer, excludedIndexes); + CPPUNIT_ASSERT(pieces.empty()); +} + void DefaultPieceStorageTest::testGetMissingFastPiece() { DefaultPieceStorage pss(dctx_, option); pss.setPieceSelector(pieceSelector_); diff --git a/test/MockPieceStorage.h b/test/MockPieceStorage.h index 82947dbd..fb1b9fc8 100644 --- a/test/MockPieceStorage.h +++ b/test/MockPieceStorage.h @@ -48,6 +48,32 @@ public: return SharedHandle(new Piece()); } + virtual void getMissingPiece + (std::vector >& pieces, + size_t minMissingBlocks, + const SharedHandle& peer, + const std::vector& excludedIndexes) + {} + + virtual void getMissingPiece + (std::vector >& pieces, + size_t minMissingBlocks, + const SharedHandle& peer) + {} + + virtual void getMissingFastPiece + (std::vector >& pieces, + size_t minMissingBlocks, + const SharedHandle& peer, + const std::vector& excludedIndexes) + {} + + virtual void getMissingFastPiece + (std::vector >& pieces, + size_t minMissingBlocks, + const SharedHandle& peer) + {} + virtual SharedHandle getMissingPiece (const SharedHandle& peer, const std::vector& excludedIndexes) {