mirror of
https://github.com/aria2/aria2.git
synced 2025-04-06 05:57:36 +03:00
2008-06-16 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>
Create only requested files and files which shares a piece with requested file in multi-torrent downloads. Directory structures are also created in the same rule. MultiDiskAdaptor::fileExists() now uses FileEntry instead of DiskWriterEntry as in the previous implementation. * src/MultiDiskWriter.cc * src/MultiFileAllocationIterator.cc * src/MultiFileAllocationIterator.h * test/MultiFileAllocationIteratorTest.cc
This commit is contained in:
parent
f9988a4ef6
commit
2f8e184fe5
5 changed files with 115 additions and 86 deletions
12
ChangeLog
12
ChangeLog
|
@ -1,3 +1,15 @@
|
||||||
|
2008-06-16 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>
|
||||||
|
|
||||||
|
Create only requested files and files which shares a piece with
|
||||||
|
requested file in multi-torrent downloads. Directory structures are
|
||||||
|
also created in the same rule.
|
||||||
|
MultiDiskAdaptor::fileExists() now uses FileEntry instead of
|
||||||
|
DiskWriterEntry as in the previous implementation.
|
||||||
|
* src/MultiDiskWriter.cc
|
||||||
|
* src/MultiFileAllocationIterator.cc
|
||||||
|
* src/MultiFileAllocationIterator.h
|
||||||
|
* test/MultiFileAllocationIteratorTest.cc
|
||||||
|
|
||||||
2008-06-14 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>
|
2008-06-14 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>
|
||||||
|
|
||||||
Commented out -lprofiler
|
Commented out -lprofiler
|
||||||
|
|
|
@ -149,20 +149,77 @@ MultiDiskAdaptor::MultiDiskAdaptor():
|
||||||
|
|
||||||
MultiDiskAdaptor::~MultiDiskAdaptor() {}
|
MultiDiskAdaptor::~MultiDiskAdaptor() {}
|
||||||
|
|
||||||
|
static SharedHandle<DiskWriterEntry> createDiskWriterEntry
|
||||||
|
(const SharedHandle<FileEntry>& fileEntry,
|
||||||
|
DiskWriterFactory& dwFactory,
|
||||||
|
bool directIOAllowed)
|
||||||
|
{
|
||||||
|
SharedHandle<DiskWriterEntry> entry(new DiskWriterEntry(fileEntry));
|
||||||
|
entry->setDiskWriter(dwFactory.newDiskWriter());
|
||||||
|
entry->getDiskWriter()->setDirectIOAllowed(directIOAllowed);
|
||||||
|
|
||||||
|
return entry;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
void MultiDiskAdaptor::resetDiskWriterEntries()
|
void MultiDiskAdaptor::resetDiskWriterEntries()
|
||||||
{
|
{
|
||||||
diskWriterEntries.clear();
|
diskWriterEntries.clear();
|
||||||
for(FileEntries::const_iterator itr = fileEntries.begin();
|
|
||||||
itr != fileEntries.end(); itr++) {
|
if(fileEntries.empty()) {
|
||||||
DiskWriterEntryHandle entry(new DiskWriterEntry(*itr));
|
return;
|
||||||
if((*itr)->isRequested()) {
|
}
|
||||||
entry->setDiskWriter(DefaultDiskWriterFactory().newDiskWriter());
|
|
||||||
} else {
|
DefaultDiskWriterFactory dwFactory;
|
||||||
SharedHandle<DiskWriter> dw(new DefaultDiskWriter());
|
if(pieceLength == 0) {
|
||||||
entry->setDiskWriter(dw);
|
for(std::deque<SharedHandle<FileEntry> >::const_iterator itr =
|
||||||
|
fileEntries.begin(); itr != fileEntries.end(); ++itr) {
|
||||||
|
if((*itr)->isRequested()) {
|
||||||
|
diskWriterEntries.push_back
|
||||||
|
(createDiskWriterEntry(*itr, dwFactory, _directIOAllowed));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
std::deque<SharedHandle<FileEntry> >::const_iterator done = fileEntries.begin();
|
||||||
|
for(std::deque<SharedHandle<FileEntry> >::const_iterator itr =
|
||||||
|
fileEntries.begin(); itr != fileEntries.end();) {
|
||||||
|
if(!(*itr)->isRequested()) {
|
||||||
|
++itr;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
off_t pieceStartOffset = ((*itr)->getOffset()/pieceLength)*pieceLength;
|
||||||
|
std::deque<SharedHandle<DiskWriterEntry> >::iterator insertionPoint =
|
||||||
|
diskWriterEntries.end();
|
||||||
|
|
||||||
|
if(itr != fileEntries.begin()) {
|
||||||
|
for(std::deque<SharedHandle<FileEntry> >::const_iterator i = itr-1;
|
||||||
|
i != done; --i) {
|
||||||
|
if((uint64_t)pieceStartOffset < (*i)->getOffset()+(*i)->getLength()) {
|
||||||
|
insertionPoint = diskWriterEntries.insert
|
||||||
|
(insertionPoint,
|
||||||
|
createDiskWriterEntry(*i, dwFactory, _directIOAllowed));
|
||||||
|
} else {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
diskWriterEntries.push_back
|
||||||
|
(createDiskWriterEntry(*itr, dwFactory, _directIOAllowed));
|
||||||
|
|
||||||
|
++itr;
|
||||||
|
|
||||||
|
for(; itr != fileEntries.end(); ++itr) {
|
||||||
|
if((*itr)->getOffset() < pieceStartOffset+pieceLength) {
|
||||||
|
diskWriterEntries.push_back
|
||||||
|
(createDiskWriterEntry(*itr, dwFactory, _directIOAllowed));
|
||||||
|
} else {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
done = itr-1;
|
||||||
}
|
}
|
||||||
entry->getDiskWriter()->setDirectIOAllowed(_directIOAllowed);
|
|
||||||
diskWriterEntries.push_back(entry);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -173,9 +230,9 @@ std::string MultiDiskAdaptor::getTopDirPath() const
|
||||||
|
|
||||||
void MultiDiskAdaptor::mkdir(const std::string& topDirPath) const
|
void MultiDiskAdaptor::mkdir(const std::string& topDirPath) const
|
||||||
{
|
{
|
||||||
for(FileEntries::const_iterator itr = fileEntries.begin();
|
for(std::deque<SharedHandle<DiskWriterEntry> >::const_iterator i =
|
||||||
itr != fileEntries.end(); itr++) {
|
diskWriterEntries.begin(); i != diskWriterEntries.end(); ++i) {
|
||||||
(*itr)->setupDir(topDirPath);
|
(*i)->getFileEntry()->setupDir(topDirPath);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -211,16 +268,16 @@ void MultiDiskAdaptor::openIfNot
|
||||||
void MultiDiskAdaptor::openFile()
|
void MultiDiskAdaptor::openFile()
|
||||||
{
|
{
|
||||||
_cachedTopDirPath = getTopDirPath();
|
_cachedTopDirPath = getTopDirPath();
|
||||||
mkdir(_cachedTopDirPath);
|
|
||||||
resetDiskWriterEntries();
|
resetDiskWriterEntries();
|
||||||
|
mkdir(_cachedTopDirPath);
|
||||||
// TODO we should call openIfNot here?
|
// TODO we should call openIfNot here?
|
||||||
}
|
}
|
||||||
|
|
||||||
void MultiDiskAdaptor::initAndOpenFile()
|
void MultiDiskAdaptor::initAndOpenFile()
|
||||||
{
|
{
|
||||||
_cachedTopDirPath = getTopDirPath();
|
_cachedTopDirPath = getTopDirPath();
|
||||||
mkdir(_cachedTopDirPath);
|
|
||||||
resetDiskWriterEntries();
|
resetDiskWriterEntries();
|
||||||
|
mkdir(_cachedTopDirPath);
|
||||||
// Call DiskWriterEntry::initAndOpenFile to make files truncated.
|
// Call DiskWriterEntry::initAndOpenFile to make files truncated.
|
||||||
for(DiskWriterEntries::iterator itr = diskWriterEntries.begin();
|
for(DiskWriterEntries::iterator itr = diskWriterEntries.begin();
|
||||||
itr != diskWriterEntries.end(); ++itr) {
|
itr != diskWriterEntries.end(); ++itr) {
|
||||||
|
@ -339,15 +396,13 @@ ssize_t MultiDiskAdaptor::readData(unsigned char* data, size_t len, off_t offset
|
||||||
|
|
||||||
bool MultiDiskAdaptor::fileExists()
|
bool MultiDiskAdaptor::fileExists()
|
||||||
{
|
{
|
||||||
if(diskWriterEntries.empty()) {
|
|
||||||
resetDiskWriterEntries();
|
|
||||||
}
|
|
||||||
// Don't use _cachedTopDirPath because they are initialized after opening files.
|
// Don't use _cachedTopDirPath because they are initialized after opening files.
|
||||||
// This method could be called before opening files.
|
// This method could be called before opening files.
|
||||||
std::string topDirPath = getTopDirPath();
|
std::string topDirPath = getTopDirPath();
|
||||||
for(DiskWriterEntries::iterator itr = diskWriterEntries.begin();
|
for(std::deque<SharedHandle<FileEntry> >::iterator i =
|
||||||
itr != diskWriterEntries.end(); ++itr) {
|
fileEntries.begin(); i != fileEntries.end(); ++i) {
|
||||||
if((*itr)->fileExists(topDirPath)) {
|
|
||||||
|
if(File(topDirPath+"/"+(*i)->getPath()).isFile()) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -42,11 +42,9 @@ namespace aria2 {
|
||||||
|
|
||||||
MultiFileAllocationIterator::MultiFileAllocationIterator(MultiDiskAdaptor* diskAdaptor):
|
MultiFileAllocationIterator::MultiFileAllocationIterator(MultiDiskAdaptor* diskAdaptor):
|
||||||
_diskAdaptor(diskAdaptor),
|
_diskAdaptor(diskAdaptor),
|
||||||
|
_entries(_diskAdaptor->diskWriterEntries),
|
||||||
_offset(0)
|
_offset(0)
|
||||||
{
|
{}
|
||||||
makeDiskWriterEntries(_entries, diskAdaptor->diskWriterEntries,
|
|
||||||
diskAdaptor->getPieceLength());
|
|
||||||
}
|
|
||||||
|
|
||||||
MultiFileAllocationIterator::~MultiFileAllocationIterator() {}
|
MultiFileAllocationIterator::~MultiFileAllocationIterator() {}
|
||||||
|
|
||||||
|
@ -106,41 +104,4 @@ const DiskWriterEntries& MultiFileAllocationIterator::getDiskWriterEntries() con
|
||||||
return _entries;
|
return _entries;
|
||||||
}
|
}
|
||||||
|
|
||||||
void MultiFileAllocationIterator::makeDiskWriterEntries
|
|
||||||
(std::deque<SharedHandle<DiskWriterEntry> >& dwEntries,
|
|
||||||
const DiskWriterEntries& srcEntries, size_t pieceLength) const
|
|
||||||
{
|
|
||||||
if(pieceLength == 0) {
|
|
||||||
for(DiskWriterEntries::const_iterator itr = srcEntries.begin(); itr != srcEntries.end(); ++itr) {
|
|
||||||
if((*itr)->getFileEntry()->isRequested()) {
|
|
||||||
dwEntries.push_back(*itr);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
DiskWriterEntries temp(srcEntries);
|
|
||||||
{
|
|
||||||
SharedHandle<FileEntry> f(new FileEntry());
|
|
||||||
SharedHandle<DiskWriterEntry> e(new DiskWriterEntry(f));
|
|
||||||
temp.push_front(e);
|
|
||||||
}
|
|
||||||
DiskWriterEntries::const_iterator done = temp.begin();
|
|
||||||
for(DiskWriterEntries::const_iterator itr = temp.begin()+1; itr != temp.end(); ++itr) {
|
|
||||||
const FileEntryHandle& fileEntry = (*itr)->getFileEntry();
|
|
||||||
if(!fileEntry->isRequested()) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
off_t pieceStartOffset = (fileEntry->getOffset()/pieceLength)*pieceLength;
|
|
||||||
for(DiskWriterEntries::const_iterator i = itr-1; i != done; --i) {
|
|
||||||
if((uint64_t)pieceStartOffset < (*i)->getFileEntry()->getOffset()+(*i)->getFileEntry()->getLength()) {
|
|
||||||
dwEntries.push_back(*i);
|
|
||||||
} else {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
dwEntries.push_back(*itr);
|
|
||||||
done = itr;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
} // namespace aria2
|
} // namespace aria2
|
||||||
|
|
|
@ -52,10 +52,6 @@ private:
|
||||||
SharedHandle<SingleFileAllocationIterator> _fileAllocationIterator;
|
SharedHandle<SingleFileAllocationIterator> _fileAllocationIterator;
|
||||||
off_t _offset;
|
off_t _offset;
|
||||||
|
|
||||||
void makeDiskWriterEntries
|
|
||||||
(std::deque<SharedHandle<DiskWriterEntry> >& dwEntries,
|
|
||||||
const std::deque<SharedHandle<DiskWriterEntry> >& srcEntries,
|
|
||||||
size_t pieceLength) const;
|
|
||||||
public:
|
public:
|
||||||
MultiFileAllocationIterator(MultiDiskAdaptor* diskAdaptor);
|
MultiFileAllocationIterator(MultiDiskAdaptor* diskAdaptor);
|
||||||
|
|
||||||
|
|
|
@ -3,6 +3,7 @@
|
||||||
#include "MultiDiskAdaptor.h"
|
#include "MultiDiskAdaptor.h"
|
||||||
#include "FileEntry.h"
|
#include "FileEntry.h"
|
||||||
#include "Exception.h"
|
#include "Exception.h"
|
||||||
|
#include "array_fun.h"
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
#include <iostream>
|
#include <iostream>
|
||||||
#include <cppunit/extensions/HelperMacros.h>
|
#include <cppunit/extensions/HelperMacros.h>
|
||||||
|
@ -31,27 +32,28 @@ void MultiFileAllocationIteratorTest::testMakeDiskWriterEntries()
|
||||||
{
|
{
|
||||||
SharedHandle<FileEntry> fs[] = {
|
SharedHandle<FileEntry> fs[] = {
|
||||||
SharedHandle<FileEntry>(new FileEntry("file1", 1536, 0)),
|
SharedHandle<FileEntry>(new FileEntry("file1", 1536, 0)),
|
||||||
SharedHandle<FileEntry>(new FileEntry("file2", 2048, 1536)),
|
SharedHandle<FileEntry>(new FileEntry("file2", 2048, 1536)),// req no
|
||||||
SharedHandle<FileEntry>(new FileEntry("file3", 1024, 3584)),
|
SharedHandle<FileEntry>(new FileEntry("file3", 1024, 3584)),
|
||||||
SharedHandle<FileEntry>(new FileEntry("file4", 1024, 4608)),
|
SharedHandle<FileEntry>(new FileEntry("file4", 1024, 4608)),// req no
|
||||||
SharedHandle<FileEntry>(new FileEntry("file5", 1024, 5632)),
|
SharedHandle<FileEntry>(new FileEntry("file5", 1024, 5632)),// req no
|
||||||
SharedHandle<FileEntry>(new FileEntry("file6", 1024, 6656)),
|
SharedHandle<FileEntry>(new FileEntry("file6", 1024, 6656)),// req no
|
||||||
SharedHandle<FileEntry>(new FileEntry("file7", 256, 7680)),
|
SharedHandle<FileEntry>(new FileEntry("file7", 256, 7680)),// req no
|
||||||
SharedHandle<FileEntry>(new FileEntry("file8", 768, 7936)),
|
SharedHandle<FileEntry>(new FileEntry("file8", 255, 7936)),
|
||||||
SharedHandle<FileEntry>(new FileEntry("file9", 256, 8704)),
|
SharedHandle<FileEntry>(new FileEntry("file9", 1025, 8191)),// req no
|
||||||
SharedHandle<FileEntry>(new FileEntry("fileA", 256, 8960)),
|
SharedHandle<FileEntry>(new FileEntry("fileA", 1024, 9216)),// req no
|
||||||
|
SharedHandle<FileEntry>(new FileEntry("fileB", 1024, 10240)),
|
||||||
};
|
};
|
||||||
fs[1]->setRequested(false);
|
fs[1]->setRequested(false); // file2
|
||||||
fs[3]->setRequested(false);
|
fs[3]->setRequested(false); // file4
|
||||||
fs[4]->setRequested(false);
|
fs[4]->setRequested(false); // file5
|
||||||
fs[5]->setRequested(false);
|
fs[5]->setRequested(false); // file6
|
||||||
fs[6]->setRequested(false);
|
fs[6]->setRequested(false); // file7
|
||||||
fs[8]->setRequested(false);
|
fs[8]->setRequested(false); // file9
|
||||||
fs[9]->setRequested(false);
|
fs[9]->setRequested(false); // fileA
|
||||||
|
|
||||||
std::string storeDir = "/tmp/aria2_MultiFileAllocationIteratorTest_testMakeDiskWriterEntries";
|
std::string storeDir = "/tmp/aria2_MultiFileAllocationIteratorTest_testMakeDiskWriterEntries";
|
||||||
SharedHandle<MultiDiskAdaptor> diskAdaptor(new MultiDiskAdaptor());
|
SharedHandle<MultiDiskAdaptor> diskAdaptor(new MultiDiskAdaptor());
|
||||||
diskAdaptor->setFileEntries(std::deque<SharedHandle<FileEntry> >(&fs[0], &fs[10]));
|
diskAdaptor->setFileEntries(std::deque<SharedHandle<FileEntry> >(&fs[0], &fs[arrayLength(fs)]));
|
||||||
diskAdaptor->setPieceLength(1024);
|
diskAdaptor->setPieceLength(1024);
|
||||||
diskAdaptor->setStoreDir(storeDir);
|
diskAdaptor->setStoreDir(storeDir);
|
||||||
diskAdaptor->openFile();
|
diskAdaptor->openFile();
|
||||||
|
@ -63,7 +65,7 @@ void MultiFileAllocationIteratorTest::testMakeDiskWriterEntries()
|
||||||
|
|
||||||
std::sort(entries.begin(), entries.end());
|
std::sort(entries.begin(), entries.end());
|
||||||
|
|
||||||
CPPUNIT_ASSERT_EQUAL((size_t)6, entries.size());
|
CPPUNIT_ASSERT_EQUAL((size_t)8, entries.size());
|
||||||
|
|
||||||
CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/file1"), entries[0]->getFilePath(storeDir));
|
CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/file1"), entries[0]->getFilePath(storeDir));
|
||||||
CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/file2"), entries[1]->getFilePath(storeDir));
|
CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/file2"), entries[1]->getFilePath(storeDir));
|
||||||
|
@ -71,6 +73,8 @@ void MultiFileAllocationIteratorTest::testMakeDiskWriterEntries()
|
||||||
CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/file6"), entries[3]->getFilePath(storeDir));
|
CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/file6"), entries[3]->getFilePath(storeDir));
|
||||||
CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/file7"), entries[4]->getFilePath(storeDir));
|
CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/file7"), entries[4]->getFilePath(storeDir));
|
||||||
CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/file8"), entries[5]->getFilePath(storeDir));
|
CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/file8"), entries[5]->getFilePath(storeDir));
|
||||||
|
CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/file9"), entries[6]->getFilePath(storeDir));
|
||||||
|
CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/fileB"), entries[7]->getFilePath(storeDir));
|
||||||
}
|
}
|
||||||
|
|
||||||
void MultiFileAllocationIteratorTest::testAllocate()
|
void MultiFileAllocationIteratorTest::testAllocate()
|
||||||
|
@ -145,9 +149,10 @@ void MultiFileAllocationIteratorTest::testAllocate()
|
||||||
CPPUNIT_ASSERT_EQUAL((uint64_t)length1, File(dir+"/"+topDir+"/"+fname1).size());
|
CPPUNIT_ASSERT_EQUAL((uint64_t)length1, File(dir+"/"+topDir+"/"+fname1).size());
|
||||||
CPPUNIT_ASSERT_EQUAL((uint64_t)length2, File(dir+"/"+topDir+"/"+fname2).size());
|
CPPUNIT_ASSERT_EQUAL((uint64_t)length2, File(dir+"/"+topDir+"/"+fname2).size());
|
||||||
CPPUNIT_ASSERT_EQUAL((uint64_t)length3, File(dir+"/"+topDir+"/"+fname3).size());
|
CPPUNIT_ASSERT_EQUAL((uint64_t)length3, File(dir+"/"+topDir+"/"+fname3).size());
|
||||||
CPPUNIT_ASSERT_EQUAL(0ULL, File(dir+"/"+topDir+"/"+fname4).size());
|
CPPUNIT_ASSERT(!File(dir+"/"+topDir+"/"+fname4).isFile());
|
||||||
|
|
||||||
CPPUNIT_ASSERT_EQUAL((uint64_t)length5, File(dir+"/"+topDir+"/"+fname5).size());
|
CPPUNIT_ASSERT_EQUAL((uint64_t)length5, File(dir+"/"+topDir+"/"+fname5).size());
|
||||||
CPPUNIT_ASSERT_EQUAL(0ULL, File(dir+"/"+topDir+"/"+fname6).size());
|
CPPUNIT_ASSERT(!File(dir+"/"+topDir+"/"+fname6).isFile());
|
||||||
|
|
||||||
} catch(Exception& e) {
|
} catch(Exception& e) {
|
||||||
CPPUNIT_FAIL(e.stackTrace());
|
CPPUNIT_FAIL(e.stackTrace());
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue