mirrors/aria2
1
0
Fork 0
mirror of https://github.com/aria2/aria2.git synced 2025-04-05 05:27:38 +03:00
Code Issues Projects Releases Packages Wiki Activity

Added --disk-cache option

Browse source 
This option enables disk cache. If SIZE is 0, the disk cache is
disabled. This feature caches the downloaded data in memory, which
grows to at most SIZE bytes. The cache storage is created for aria2
instance and shared by all downloads. The one advantage of the disk
cache is reduce the disk seek time because the data is written in
larger unit and it is reordered by the offset of the file. If the
underlying file is heavily fragmented it is not the case.
This commit is contained in:
Tatsuhiro Tsujikawa 2012-11-27 22:04:59 +09:00
parent 8ac433a8e9
commit f314719618
33 changed files with 1062 additions and 57 deletions
Download patch file Download diff file Expand all files Collapse all files

4
test/Makefile.am
View file

@ -85,7 +85,9 @@ aria2c_SOURCES = AllTest.cc\
ParamedStringTest.cc\
RpcHelperTest.cc\
AbstractCommandTest.cc\
SinkStreamFilterTest.cc
SinkStreamFilterTest.cc\
WrDiskCacheTest.cc\
WrDiskCacheEntryTest.cc
if ENABLE_XML_RPC
aria2c_SOURCES += XmlRpcRequestParserControllerTest.cc

6
test/MockPieceStorage.h
View file

@ -226,6 +226,12 @@ public:
return diskAdaptor;
}
virtual WrDiskCache* getWrDiskCache() {
return 0;
}
virtual void flushWrDiskCacheEntry() {}
void setDiskAdaptor(const SharedHandle<DiskAdaptor>& adaptor) {
this->diskAdaptor = adaptor;
}

68
test/PieceTest.cc
View file

@ -5,6 +5,9 @@
#include <cppunit/extensions/HelperMacros.h>
#include "util.h"
#include "DirectDiskAdaptor.h"
#include "ByteArrayDiskWriter.h"
#include "WrDiskCache.h"
namespace aria2 {
@ -13,24 +16,33 @@ class PieceTest:public CppUnit::TestFixture {
CPPUNIT_TEST_SUITE(PieceTest);
CPPUNIT_TEST(testCompleteBlock);
CPPUNIT_TEST(testGetCompletedLength);
CPPUNIT_TEST(testFlushWrCache);
#ifdef ENABLE_MESSAGE_DIGEST
CPPUNIT_TEST(testGetDigestWithWrCache);
CPPUNIT_TEST(testUpdateHash);
#endif // ENABLE_MESSAGE_DIGEST
CPPUNIT_TEST_SUITE_END();
private:
SharedHandle<DirectDiskAdaptor> adaptor_;
SharedHandle<ByteArrayDiskWriter> writer_;
public:
void setUp() {}
void setUp()
{
adaptor_.reset(new DirectDiskAdaptor());
writer_.reset(new ByteArrayDiskWriter());
adaptor_->setDiskWriter(writer_);
}
void testCompleteBlock();
void testGetCompletedLength();
void testFlushWrCache();
#ifdef ENABLE_MESSAGE_DIGEST
void testGetDigestWithWrCache();
void testUpdateHash();
#endif // ENABLE_MESSAGE_DIGEST
@ -62,8 +74,58 @@ void PieceTest::testGetCompletedLength()
CPPUNIT_ASSERT_EQUAL(blockLength*3+100, p.getCompletedLength());
}
void PieceTest::testFlushWrCache()
{
unsigned char* data;
Piece p(0, 1024);
WrDiskCache dc(64);
p.initWrCache(&dc, adaptor_);
data = new unsigned char[3];
memcpy(data, "foo", 3);
p.updateWrCache(&dc, data, 0, 3, 0);
data = new unsigned char[4];
memcpy(data, " bar", 4);
p.updateWrCache(&dc, data, 0, 4, 3);
p.flushWrCache(&dc);
CPPUNIT_ASSERT_EQUAL(std::string("foo bar"), writer_->getString());
data = new unsigned char[3];
memcpy(data, "foo", 3);
p.updateWrCache(&dc, data, 0, 3, 0);
CPPUNIT_ASSERT_EQUAL((size_t)3, dc.getSize());
p.clearWrCache(&dc);
CPPUNIT_ASSERT_EQUAL((size_t)0, dc.getSize());
p.releaseWrCache(&dc);
CPPUNIT_ASSERT(!p.getWrDiskCacheEntry());
}
#ifdef ENABLE_MESSAGE_DIGEST
void PieceTest::testGetDigestWithWrCache()
{
unsigned char* data;
Piece p(0, 26);
p.setHashType("sha-1");
WrDiskCache dc(64);
// 012345678901234567890123456
writer_->setString("abcde...ijklmnopq...uvwx.z");
p.initWrCache(&dc, adaptor_);
data = new unsigned char[3];
memcpy(data, "fgh", 3);
p.updateWrCache(&dc, data, 0, 3, 5);
data = new unsigned char[3];
memcpy(data, "rst", 3);
p.updateWrCache(&dc, data, 0, 3, 17);
data = new unsigned char[1];
memcpy(data, "y", 1);
p.updateWrCache(&dc, data, 0, 1, 24);
CPPUNIT_ASSERT_EQUAL
(std::string("32d10c7b8cf96570ca04ce37f2a19d84240d3a89"),
util::toHex(p.getDigestWithWrCache(p.getLength(), adaptor_)));
}
void PieceTest::testUpdateHash()
{
Piece p(0, 16, 2*1024*1024);

4
test/SinkStreamFilterTest.cc
View file

@ -54,9 +54,7 @@ public:
void setUp()
{
writer_.reset(new ByteArrayDiskWriter());
sinkFilter_.reset(new SinkStreamFilter());
filter_.reset(new SinkStreamFilter(sinkFilter_));
sinkFilter_->init();
filter_.reset(new SinkStreamFilter());
filter_->init();
segment_.reset(new MockSegment2(16));
}

14
test/TestUtil.cc
View file

@ -91,4 +91,18 @@ std::string fileHexDigest
}
#endif // ENABLE_MESSAGE_DIGEST
WrDiskCacheEntry::DataCell* createDataCell(int64_t goff,
const char* data,
size_t offset)
{
WrDiskCacheEntry::DataCell* cell = new WrDiskCacheEntry::DataCell();
cell->goff = goff;
size_t len = strlen(data);
cell->data = new unsigned char[len];
memcpy(cell->data, data, len);
cell->offset = offset;
cell->len = len;
return cell;
}
} // namespace aria2

5
test/TestUtil.h
View file

@ -4,6 +4,7 @@
#include "SharedHandle.h"
#include "Cookie.h"
#include "WrDiskCacheEntry.h"
namespace aria2 {
@ -50,4 +51,8 @@ std::string fileHexDigest
(const SharedHandle<MessageDigest>& ctx, const std::string& filename);
#endif // ENABLE_MESSAGE_DIGEST
WrDiskCacheEntry::DataCell* createDataCell(int64_t goff,
const char* data,
size_t offset = 0);
} // namespace aria2

55
test/WrDiskCacheEntryTest.cc Normal file
View file

@ -0,0 +1,55 @@
#include "WrDiskCacheEntry.h"
#include <cstring>
#include <cppunit/extensions/HelperMacros.h>
#include "TestUtil.h"
#include "DirectDiskAdaptor.h"
#include "ByteArrayDiskWriter.h"
namespace aria2 {
class WrDiskCacheEntryTest:public CppUnit::TestFixture {
CPPUNIT_TEST_SUITE(WrDiskCacheEntryTest);
CPPUNIT_TEST(testWriteToDisk);
CPPUNIT_TEST(testClear);
CPPUNIT_TEST_SUITE_END();
SharedHandle<DirectDiskAdaptor> adaptor_;
SharedHandle<ByteArrayDiskWriter> writer_;
public:
void setUp()
{
adaptor_.reset(new DirectDiskAdaptor());
writer_.reset(new ByteArrayDiskWriter());
adaptor_->setDiskWriter(writer_);
}
void testWriteToDisk();
void testClear();
};
CPPUNIT_TEST_SUITE_REGISTRATION( WrDiskCacheEntryTest );
void WrDiskCacheEntryTest::testWriteToDisk()
{
WrDiskCacheEntry e(adaptor_);
e.cacheData(createDataCell(0, "??01234567", 2));
e.cacheData(createDataCell(8, "890"));
e.writeToDisk();
CPPUNIT_ASSERT_EQUAL((size_t)0, e.getSize());
CPPUNIT_ASSERT_EQUAL(std::string("01234567890"), writer_->getString());
}
void WrDiskCacheEntryTest::testClear()
{
WrDiskCacheEntry e(adaptor_);
e.cacheData(createDataCell(0, "foo"));
e.clear();
CPPUNIT_ASSERT_EQUAL((size_t)0, e.getSize());
CPPUNIT_ASSERT_EQUAL(std::string(), writer_->getString());
}
} // namespace aria2

74
test/WrDiskCacheTest.cc Normal file
View file

@ -0,0 +1,74 @@
#include "WrDiskCache.h"
#include <cstring>
#include <cppunit/extensions/HelperMacros.h>
#include "TestUtil.h"
#include "DirectDiskAdaptor.h"
#include "ByteArrayDiskWriter.h"
namespace aria2 {
class WrDiskCacheTest:public CppUnit::TestFixture {
CPPUNIT_TEST_SUITE(WrDiskCacheTest);
CPPUNIT_TEST(testAdd);
CPPUNIT_TEST_SUITE_END();
SharedHandle<DirectDiskAdaptor> adaptor_;
SharedHandle<ByteArrayDiskWriter> writer_;
public:
void setUp()
{
adaptor_.reset(new DirectDiskAdaptor());
writer_.reset(new ByteArrayDiskWriter());
adaptor_->setDiskWriter(writer_);
}
void testAdd();
};
CPPUNIT_TEST_SUITE_REGISTRATION( WrDiskCacheTest );
void WrDiskCacheTest::testAdd()
{
WrDiskCache dc(20);
CPPUNIT_ASSERT_EQUAL((size_t)0, dc.getSize());
WrDiskCacheEntry e1(adaptor_);
e1.cacheData(createDataCell(0, "who knows?"));
CPPUNIT_ASSERT(dc.add(&e1));
CPPUNIT_ASSERT_EQUAL((size_t)10, dc.getSize());
WrDiskCacheEntry e2(adaptor_);
e2.cacheData(createDataCell(21, "seconddata"));
CPPUNIT_ASSERT(dc.add(&e2));
CPPUNIT_ASSERT_EQUAL((size_t)20, dc.getSize());
WrDiskCacheEntry e3(adaptor_);
e3.cacheData(createDataCell(10, "hello"));
CPPUNIT_ASSERT(dc.add(&e3));
CPPUNIT_ASSERT_EQUAL((size_t)15, dc.getSize());
// e1 is flushed to the disk
CPPUNIT_ASSERT_EQUAL(std::string("who knows?"), writer_->getString());
CPPUNIT_ASSERT_EQUAL((size_t)0, e1.getSize());
e3.cacheData(createDataCell(15, " world"));
CPPUNIT_ASSERT(dc.update(&e3, 6));
// e3 is flushed to the disk
CPPUNIT_ASSERT_EQUAL(std::string("who knows?hello world"),
writer_->getString());
CPPUNIT_ASSERT_EQUAL((size_t)0, e3.getSize());
CPPUNIT_ASSERT_EQUAL((size_t)10, dc.getSize());
e2.cacheData(createDataCell(31, "01234567890"));
CPPUNIT_ASSERT(dc.update(&e2, 11));
// e2 is flushed to the disk
CPPUNIT_ASSERT_EQUAL(std::string("who knows?hello worldseconddata01234567890"),
writer_->getString());
CPPUNIT_ASSERT_EQUAL((size_t)0, e2.getSize());
CPPUNIT_ASSERT_EQUAL((size_t)0, dc.getSize());
}
} // namespace aria2