From 6bc233f414b74bf3acf4abf5d5655c236e78ae41 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 24 Sep 2008 17:01:57 +0000 Subject: [PATCH] 2008-09-25 Tatsuhiro Tsujikawa Issue PWD command first and get working directory and use it as a prefix for CWD command. * src/DownloadEngine.cc * src/DownloadEngine.h * src/FtpConnection.cc * src/FtpConnection.h * src/FtpFinishDownloadCommand.cc * src/FtpInitiateConnectionCommand.cc * src/FtpNegotiationCommand.cc * src/FtpNegotiationCommand.h * test/FtpConnectionTest.cc --- ChangeLog | 14 ++++ src/DownloadEngine.cc | 124 +++++++++++++++++++++++----- src/DownloadEngine.h | 33 +++++++- src/FtpConnection.cc | 51 +++++++++++- src/FtpConnection.h | 8 ++ src/FtpFinishDownloadCommand.cc | 5 +- src/FtpInitiateConnectionCommand.cc | 10 ++- src/FtpNegotiationCommand.cc | 40 ++++++++- src/FtpNegotiationCommand.h | 7 +- test/FtpConnectionTest.cc | 78 +++++++++++++++++ 10 files changed, 340 insertions(+), 30 deletions(-) diff --git a/ChangeLog b/ChangeLog index 3d55ee4c..64ed2c63 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,17 @@ +2008-09-25 Tatsuhiro Tsujikawa + + Issue PWD command first and get working directory and use it as a prefix + for CWD command. + * src/DownloadEngine.cc + * src/DownloadEngine.h + * src/FtpConnection.cc + * src/FtpConnection.h + * src/FtpFinishDownloadCommand.cc + * src/FtpInitiateConnectionCommand.cc + * src/FtpNegotiationCommand.cc + * src/FtpNegotiationCommand.h + * test/FtpConnectionTest.cc + 2008-09-25 Tatsuhiro Tsujikawa Removed default user/pass for FTP user/pass, since it should not have diff --git a/src/DownloadEngine.cc b/src/DownloadEngine.cc index 48e6b464..3a4b8da3 100644 --- a/src/DownloadEngine.cc +++ b/src/DownloadEngine.cc @@ -53,6 +53,7 @@ #include "DlAbortEx.h" #include "ServerStatMan.h" #include "CookieStorage.h" +#include "A2STR.h" #include #include #include @@ -891,33 +892,106 @@ SharedHandle DownloadEngine::getCookieStorage() const return _cookieStorage; } -void DownloadEngine::poolSocket(const std::string& ipaddr, uint16_t port, - const SharedHandle& sock, - time_t timeout) +void DownloadEngine::poolSocket(const std::string& ipaddr, + uint16_t port, + const SocketPoolEntry& entry) { std::string addr = ipaddr+":"+Util::uitos(port); logger->info("Pool socket for %s", addr.c_str()); - - SocketPoolEntry e(sock, timeout); - std::multimap::value_type p(addr, e); + std::multimap::value_type p(addr, entry); _socketPool.insert(p); + + if(_lastSocketPoolScan.elapsed(60)) { + std::multimap newPool; + logger->debug("Scaning SocketPool and erasing timed out entry."); + _lastSocketPoolScan.reset(); + for(std::multimap::iterator i = + _socketPool.begin(); i != _socketPool.end(); ++i) { + if(!(*i).second.isTimeout()) { + newPool.insert(*i); + } + } + logger->debug("%zu entries removed.", _socketPool.size()-newPool.size()); + _socketPool = newPool; + } +} + +void DownloadEngine::poolSocket +(const std::string& ipaddr, + uint16_t port, + const SharedHandle& sock, + const std::map& options, + time_t timeout) +{ + SocketPoolEntry e(sock, options, timeout); + poolSocket(ipaddr, port, e); +} + +void DownloadEngine::poolSocket +(const std::string& ipaddr, + uint16_t port, + const SharedHandle& sock, + time_t timeout) +{ + SocketPoolEntry e(sock, std::map(), timeout); + poolSocket(ipaddr, port, e); +} + +std::multimap::iterator +DownloadEngine::findSocketPoolEntry(const std::string& ipaddr, uint16_t port) +{ + std::string addr = ipaddr+":"+Util::uitos(port); + std::pair::iterator, + std::multimap::iterator> range = + _socketPool.equal_range(addr); + for(std::multimap::iterator i = range.first; + i != range.second; ++i) { + const SocketPoolEntry& e = (*i).second; + if(!e.isTimeout()) { + logger->info("Found socket for %s", addr.c_str()); + return i; + } + } + return _socketPool.end(); } SharedHandle DownloadEngine::popPooledSocket(const std::string& ipaddr, uint16_t port) { SharedHandle s; - std::string addr = ipaddr+":"+Util::uitos(port); + std::multimap::iterator i = + findSocketPoolEntry(ipaddr, port); + if(i != _socketPool.end()) { + s = (*i).second.getSocket(); + _socketPool.erase(i); + } + return s; +} - std::multimap::iterator first = _socketPool.find(addr); - - for(std::multimap::iterator i = first; - i != _socketPool.end() && (*i).first == addr; ++i) { - const SocketPoolEntry& e = (*i).second; - if(!e.isTimeout()) { - logger->info("Reuse socket for %s", addr.c_str()); - s = e.getSocket(); - _socketPool.erase(first, ++i); +SharedHandle +DownloadEngine::popPooledSocket(std::map& options, + const std::string& ipaddr, uint16_t port) +{ + SharedHandle s; + std::multimap::iterator i = + findSocketPoolEntry(ipaddr, port); + if(i != _socketPool.end()) { + s = (*i).second.getSocket(); + options = (*i).second.getOptions(); + _socketPool.erase(i); + } + return s; +} + +SharedHandle +DownloadEngine::popPooledSocket +(const std::deque& ipaddrs, uint16_t port) +{ + SharedHandle s; + for(std::deque::const_iterator i = ipaddrs.begin(); + i != ipaddrs.end(); ++i) { + s = popPooledSocket(*i, port); + if(!s.isNull()) { break; } } @@ -926,22 +1000,26 @@ DownloadEngine::popPooledSocket(const std::string& ipaddr, uint16_t port) SharedHandle DownloadEngine::popPooledSocket -(const std::deque& ipaddrs, uint16_t port) +(std::map& options, + const std::deque& ipaddrs, uint16_t port) { + SharedHandle s; for(std::deque::const_iterator i = ipaddrs.begin(); i != ipaddrs.end(); ++i) { - SharedHandle s = popPooledSocket(*i, port); + s = popPooledSocket(options, *i, port); if(!s.isNull()) { - return s; + break; } } - return SharedHandle(); + return s; } DownloadEngine::SocketPoolEntry::SocketPoolEntry (const SharedHandle& socket, + const std::map& options, time_t timeout): _socket(socket), + _options(options), _timeout(timeout) {} DownloadEngine::SocketPoolEntry::~SocketPoolEntry() {} @@ -956,4 +1034,10 @@ SharedHandle DownloadEngine::SocketPoolEntry::getSocket() const return _socket; } +const std::map& +DownloadEngine::SocketPoolEntry::getOptions() const +{ + return _options; +} + } // namespace aria2 diff --git a/src/DownloadEngine.h b/src/DownloadEngine.h index 61b64bfb..8a20493e 100644 --- a/src/DownloadEngine.h +++ b/src/DownloadEngine.h @@ -44,6 +44,7 @@ #ifdef ENABLE_ASYNC_DNS # include "AsyncNameResolver.h" #endif // ENABLE_ASYNC_DNS +#include #include #include #ifdef HAVE_EPOLL @@ -261,11 +262,14 @@ private: private: SharedHandle _socket; + std::map _options; + time_t _timeout; Time _registeredTime; public: SocketPoolEntry(const SharedHandle& socket, + const std::map& option, time_t timeout); ~SocketPoolEntry(); @@ -273,11 +277,15 @@ private: bool isTimeout() const; SharedHandle getSocket() const; + + const std::map& getOptions() const; }; // key = IP address:port, value = SocketPoolEntry std::multimap _socketPool; + Time _lastSocketPoolScan; + bool _noWait; std::deque _routineCommands; @@ -294,6 +302,13 @@ private: void onEndOfRun(); void afterEachIteration(); + + void poolSocket(const std::string& ipaddr, + uint16_t port, + const SocketPoolEntry& entry); + + std::multimap::iterator + findSocketPoolEntry(const std::string& ipaddr, uint16_t port); public: std::deque commands; SharedHandle _requestGroupMan; @@ -364,15 +379,31 @@ public: void addRoutineCommand(Command* command); void poolSocket(const std::string& ipaddr, uint16_t port, - const SharedHandle& sock, time_t timeout = 15); + const SharedHandle& sock, + const std::map& options, + time_t timeout = 15); + void poolSocket(const std::string& ipaddr, uint16_t port, + const SharedHandle& sock, + time_t timeout = 15); + SharedHandle popPooledSocket(const std::string& ipaddr, uint16_t port); + SharedHandle popPooledSocket + (std::map& options, + const std::string& ipaddr, + uint16_t port); SharedHandle popPooledSocket(const std::deque& ipaddrs, uint16_t port); + SharedHandle + popPooledSocket + (std::map& options, + const std::deque& ipaddrs, + uint16_t port); + SharedHandle getCookieStorage() const; }; diff --git a/src/FtpConnection.cc b/src/FtpConnection.cc index 35a437b9..09cdd6f6 100644 --- a/src/FtpConnection.cc +++ b/src/FtpConnection.cc @@ -61,7 +61,8 @@ FtpConnection::FtpConnection(int32_t cuid, const SocketHandle& socket, const RequestHandle& req, const Option* op): cuid(cuid), socket(socket), req(req), option(op), logger(LogFactory::getInstance()), - _socketBuffer(socket) {} + _socketBuffer(socket), + _baseWorkingDir("/") {} FtpConnection::~FtpConnection() {} @@ -108,10 +109,25 @@ bool FtpConnection::sendType() return _socketBuffer.sendBufferIsEmpty(); } +bool FtpConnection::sendPwd() +{ + if(_socketBuffer.sendBufferIsEmpty()) { + std::string request = "PWD\r\n"; + logger->info(MSG_SENDING_REQUEST, cuid, request.c_str()); + _socketBuffer.feedSendBuffer(request); + } + _socketBuffer.send(); + return _socketBuffer.sendBufferIsEmpty(); +} + bool FtpConnection::sendCwd() { if(_socketBuffer.sendBufferIsEmpty()) { - std::string request = "CWD "+Util::urldecode(req->getDir())+"\r\n"; + logger->info("CUID#%d - Using base working directory '%s'", + cuid, _baseWorkingDir.c_str()); + std::string request = "CWD "+ + (_baseWorkingDir == "/" ? "" : _baseWorkingDir)+ + Util::urldecode(req->getDir())+"\r\n"; logger->info(MSG_SENDING_REQUEST, cuid, request.c_str()); _socketBuffer.feedSendBuffer(request); } @@ -380,4 +396,35 @@ unsigned int FtpConnection::receivePasvResponse(std::pair } } +unsigned int FtpConnection::receivePwdResponse(std::string& pwd) +{ + std::pair response; + if(bulkReceiveResponse(response)) { + if(response.first == 257) { + std::string::size_type first; + std::string::size_type last; + + if((first = response.second.find("\"")) != std::string::npos && + (last = response.second.find("\"", ++first)) != std::string::npos) { + pwd = response.second.substr(first, last-first); + } else { + throw DlAbortEx(EX_INVALID_RESPONSE); + } + } + return response.first; + } else { + return 0; + } +} + +void FtpConnection::setBaseWorkingDir(const std::string& baseWorkingDir) +{ + _baseWorkingDir = baseWorkingDir; +} + +const std::string& FtpConnection::getBaseWorkingDir() const +{ + return _baseWorkingDir; +} + } // namespace aria2 diff --git a/src/FtpConnection.h b/src/FtpConnection.h index b88bdc0e..235a905f 100644 --- a/src/FtpConnection.h +++ b/src/FtpConnection.h @@ -62,6 +62,8 @@ private: SocketBuffer _socketBuffer; + std::string _baseWorkingDir; + unsigned int getStatus(const std::string& response) const; std::string::size_type findEndOfResponse(unsigned int status, const std::string& buf) const; @@ -79,6 +81,7 @@ public: bool sendUser(); bool sendPass(); bool sendType(); + bool sendPwd(); bool sendCwd(); bool sendMdtm(); bool sendSize(); @@ -98,6 +101,11 @@ public: // If reply is not received yet, returns 0. unsigned int receiveMdtmResponse(Time& time); unsigned int receivePasvResponse(std::pair& dest); + unsigned int receivePwdResponse(std::string& pwd); + + void setBaseWorkingDir(const std::string& baseWorkingDir); + + const std::string& getBaseWorkingDir() const; }; } // namespace aria2 diff --git a/src/FtpFinishDownloadCommand.cc b/src/FtpFinishDownloadCommand.cc index db5848cb..ea556f35 100644 --- a/src/FtpFinishDownloadCommand.cc +++ b/src/FtpFinishDownloadCommand.cc @@ -44,6 +44,7 @@ #include "SocketCore.h" #include "RequestGroup.h" #include "Logger.h" +#include namespace aria2 { @@ -85,7 +86,9 @@ bool FtpFinishDownloadCommand::execute() e->option->getAsBool(PREF_FTP_REUSE_CONNECTION)) { std::pair peerInfo; socket->getPeerInfo(peerInfo); - e->poolSocket(peerInfo.first, peerInfo.second, socket); + std::map options; + options["baseWorkingDir"] = _ftpConnection->getBaseWorkingDir(); + e->poolSocket(peerInfo.first, peerInfo.second, socket, options); } } catch(RecoverableException& e) { logger->info(EX_EXCEPTION_CAUGHT, e); diff --git a/src/FtpInitiateConnectionCommand.cc b/src/FtpInitiateConnectionCommand.cc index e1598efa..389689e3 100644 --- a/src/FtpInitiateConnectionCommand.cc +++ b/src/FtpInitiateConnectionCommand.cc @@ -47,6 +47,7 @@ #include "prefs.h" #include "HttpConnection.h" #include "Socket.h" +#include namespace aria2 { @@ -81,17 +82,20 @@ Command* FtpInitiateConnectionCommand::createNextCommand throw DlAbortEx("ERROR"); } } else { + std::map options; SharedHandle pooledSocket = - e->popPooledSocket(resolvedAddresses, req->getPort()); + e->popPooledSocket(options, resolvedAddresses, req->getPort()); if(pooledSocket.isNull()) { - logger->info(MSG_CONNECTING_TO_SERVER, cuid, req->getHost().c_str(), req->getPort()); socket.reset(new SocketCore()); socket->establishConnection(resolvedAddresses.front(), req->getPort()); command = new FtpNegotiationCommand(cuid, req, _requestGroup, e, socket); } else { - command = new FtpNegotiationCommand(cuid, req, _requestGroup, e, pooledSocket, FtpNegotiationCommand::SEQ_SEND_CWD); + command = + new FtpNegotiationCommand(cuid, req, _requestGroup, e, pooledSocket, + FtpNegotiationCommand::SEQ_SEND_CWD, + options["baseWorkingDir"]); } } return command; diff --git a/src/FtpNegotiationCommand.cc b/src/FtpNegotiationCommand.cc index 0450250e..8882173b 100644 --- a/src/FtpNegotiationCommand.cc +++ b/src/FtpNegotiationCommand.cc @@ -59,6 +59,7 @@ #include #include #include +#include namespace aria2 { @@ -67,10 +68,12 @@ FtpNegotiationCommand::FtpNegotiationCommand(int32_t cuid, RequestGroup* requestGroup, DownloadEngine* e, const SocketHandle& s, - Seq seq): + Seq seq, + const std::string& baseWorkingDir): AbstractCommand(cuid, req, requestGroup, e, s), sequence(seq), ftp(new FtpConnection(cuid, socket, req, e->option)) { + ftp->setBaseWorkingDir(baseWorkingDir); if(seq == SEQ_RECV_GREETING) { setTimeout(e->option->getAsInt(PREF_CONNECT_TIMEOUT)); } @@ -200,6 +203,33 @@ bool FtpNegotiationCommand::recvType() { if(status != 200) { throw DlAbortEx(StringFormat(EX_BAD_STATUS, status).str()); } + sequence = SEQ_SEND_PWD; + return true; +} + +bool FtpNegotiationCommand::sendPwd() +{ + if(ftp->sendPwd()) { + disableWriteCheckSocket(); + sequence = SEQ_RECV_PWD; + } else { + setWriteCheckSocket(socket); + } + return false; +} + +bool FtpNegotiationCommand::recvPwd() +{ + std::string pwd; + unsigned int status = ftp->receivePwdResponse(pwd); + if(status == 0) { + return false; + } + if(status != 257) { + throw DlAbortEx(StringFormat(EX_BAD_STATUS, status).str()); + } + ftp->setBaseWorkingDir(pwd); + logger->info("CUID#%d - base working directory is '%s'", cuid, pwd.c_str()); sequence = SEQ_SEND_CWD; return true; } @@ -536,6 +566,10 @@ bool FtpNegotiationCommand::processSequence(const SegmentHandle& segment) { return sendType(); case SEQ_RECV_TYPE: return recvType(); + case SEQ_SEND_PWD: + return sendPwd(); + case SEQ_RECV_PWD: + return recvPwd(); case SEQ_SEND_CWD: return sendCwd(); case SEQ_RECV_CWD: @@ -582,7 +616,9 @@ void FtpNegotiationCommand::poolConnection() const e->option->getAsBool(PREF_FTP_REUSE_CONNECTION)) { std::pair peerInfo; socket->getPeerInfo(peerInfo); - e->poolSocket(peerInfo.first, peerInfo.second, socket); + std::map options; + options["baseWorkingDir"] = ftp->getBaseWorkingDir(); + e->poolSocket(peerInfo.first, peerInfo.second, socket, options); } } diff --git a/src/FtpNegotiationCommand.h b/src/FtpNegotiationCommand.h index 0d80a4f5..54155ba4 100644 --- a/src/FtpNegotiationCommand.h +++ b/src/FtpNegotiationCommand.h @@ -52,6 +52,8 @@ public: SEQ_RECV_PASS, SEQ_SEND_TYPE, SEQ_RECV_TYPE, + SEQ_SEND_PWD, + SEQ_RECV_PWD, SEQ_SEND_CWD, SEQ_RECV_CWD, SEQ_SEND_MDTM, @@ -83,6 +85,8 @@ private: bool recvPass(); bool sendType(); bool recvType(); + bool sendPwd(); + bool recvPwd(); bool sendCwd(); bool recvCwd(); bool sendMdtm(); @@ -120,7 +124,8 @@ public: RequestGroup* requestGroup, DownloadEngine* e, const SharedHandle& s, - Seq seq = SEQ_RECV_GREETING); + Seq seq = SEQ_RECV_GREETING, + const std::string& baseWorkingDir = "/"); virtual ~FtpNegotiationCommand(); }; diff --git a/test/FtpConnectionTest.cc b/test/FtpConnectionTest.cc index 5a764f8b..d9143bd9 100644 --- a/test/FtpConnectionTest.cc +++ b/test/FtpConnectionTest.cc @@ -5,6 +5,7 @@ #include "Request.h" #include "Option.h" #include "DlRetryEx.h" +#include "DlAbortEx.h" #include #include #include @@ -18,6 +19,12 @@ class FtpConnectionTest:public CppUnit::TestFixture { CPPUNIT_TEST(testReceiveResponse_overflow); CPPUNIT_TEST(testSendMdtm); CPPUNIT_TEST(testReceiveMdtmResponse); + CPPUNIT_TEST(testSendPwd); + CPPUNIT_TEST(testReceivePwdResponse); + CPPUNIT_TEST(testReceivePwdResponse_unquotedResponse); + CPPUNIT_TEST(testReceivePwdResponse_badStatus); + CPPUNIT_TEST(testSendCwd); + CPPUNIT_TEST(testSendCwd_baseWorkingDir); CPPUNIT_TEST_SUITE_END(); private: SharedHandle _serverSocket; @@ -54,6 +61,12 @@ public: void testReceiveMdtmResponse(); void testReceiveResponse(); void testReceiveResponse_overflow(); + void testSendPwd(); + void testReceivePwdResponse(); + void testReceivePwdResponse_unquotedResponse(); + void testReceivePwdResponse_badStatus(); + void testSendCwd(); + void testSendCwd_baseWorkingDir(); }; @@ -159,4 +172,69 @@ void FtpConnectionTest::testReceiveResponse_overflow() } } +void FtpConnectionTest::testSendPwd() +{ + _ftp->sendPwd(); + char data[32]; + size_t len = sizeof(data); + _serverSocket->readData(data, len); + CPPUNIT_ASSERT_EQUAL((size_t)5, len); + data[len] = '\0'; + CPPUNIT_ASSERT_EQUAL(std::string("PWD\r\n"), std::string(data)); +} + +void FtpConnectionTest::testReceivePwdResponse() +{ + std::string pwd; + _serverSocket->writeData("257 "); + CPPUNIT_ASSERT_EQUAL((unsigned int)0, _ftp->receivePwdResponse(pwd)); + CPPUNIT_ASSERT(pwd.empty()); + _serverSocket->writeData("\"/dir/to\" is your directory.\r\n"); + CPPUNIT_ASSERT_EQUAL((unsigned int)257, _ftp->receivePwdResponse(pwd)); + CPPUNIT_ASSERT_EQUAL(std::string("/dir/to"), pwd); +} + +void FtpConnectionTest::testReceivePwdResponse_unquotedResponse() +{ + std::string pwd; + _serverSocket->writeData("257 /dir/to\r\n"); + try { + _ftp->receivePwdResponse(pwd); + CPPUNIT_FAIL("exception must be thrown."); + } catch(DlAbortEx& e) { + // success + } +} + +void FtpConnectionTest::testReceivePwdResponse_badStatus() +{ + std::string pwd; + _serverSocket->writeData("500 failed\r\n"); + CPPUNIT_ASSERT_EQUAL((unsigned int)500, _ftp->receivePwdResponse(pwd)); + CPPUNIT_ASSERT(pwd.empty()); +} + +void FtpConnectionTest::testSendCwd() +{ + _ftp->sendCwd(); + char data[32]; + size_t len = sizeof(data); + _serverSocket->readData(data, len); + CPPUNIT_ASSERT_EQUAL((size_t)10, len); + data[len] = '\0'; + CPPUNIT_ASSERT_EQUAL(std::string("CWD /dir\r\n"), std::string(data)); +} + +void FtpConnectionTest::testSendCwd_baseWorkingDir() +{ + _ftp->setBaseWorkingDir("/base"); + _ftp->sendCwd(); + char data[32]; + size_t len = sizeof(data); + _serverSocket->readData(data, len); + CPPUNIT_ASSERT_EQUAL((size_t)15, len); + data[len] = '\0'; + CPPUNIT_ASSERT_EQUAL(std::string("CWD /base/dir\r\n"), std::string(data)); +} + } // namespace aria2