2008-04-13 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>

Implemented auto protocol detection.
	Now you can do:
	aria2c -Z http://host/file file1.torrent file2.metalink
	(Note: -Z option is required for auto protcol detection.)
	Then aria2c downloads 3 files simultaneously:
	1. http://host/file
	2. file1.torrent <-- read local torrent file
	3. file2.metalink <-- read local Metalink file.

	Same thing goes with -i option. Assume your uris.txt contans:
	http://host/file
	file1.torrent
	file2.metalink
	Then you can do: aria2c -i uris.txt
	(Note: -Z option is not needed if -i option is given.)
	
	* src/main.cc
	* src/ProtocolDetector.{cc, h}
	* test/ProtocolDetectorTest.cc

ChangeLog

@@ -1,3 +1,25 @@
+2008-04-13  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
+
+	Implemented auto protocol detection.
+	Now you can do:
+	aria2c -Z http://host/file file1.torrent file2.metalink
+	(Note: -Z option is required for auto protcol detection.)
+	Then aria2c downloads 3 files simultaneously:
+	1. http://host/file
+	2. file1.torrent <-- read local torrent file
+	3. file2.metalink <-- read local Metalink file.
+
+	Same thing goes with -i option. Assume your uris.txt contans:
+	http://host/file
+	file1.torrent
+	file2.metalink
+	Then you can do: aria2c -i uris.txt
+	(Note: -Z option is not needed if -i option is given.)
+	
+	* src/main.cc
+	* src/ProtocolDetector.{cc, h}
+	* test/ProtocolDetectorTest.cc
+	
 2008-04-13  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
 
         Fixed compile error without gnutls/libgcrypt/libgpg-error and openSSL

src/Makefile.am

@@ -180,7 +180,8 @@ SRCS =  Socket.h\
 	array_fun.h\
 	help_tags.h\
 	prefs.h\
-	usage_text.h
+	usage_text.h\
+	ProtocolDetector.cc ProtocolDetector.h
 
 if ENABLE_MESSAGE_DIGEST
 SRCS += IteratableChunkChecksumValidator.cc IteratableChunkChecksumValidator.h\

src/Makefile.in

@@ -403,7 +403,8 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \
 	PostDownloadHandler.h PreDownloadHandler.h SingletonHolder.h \
 	TrueRequestGroupCriteria.h a2algo.h a2functional.h a2io.h \
 	a2netcompat.h a2time.h array_fun.h help_tags.h prefs.h \
-	usage_text.h IteratableChunkChecksumValidator.cc \
+	usage_text.h ProtocolDetector.cc ProtocolDetector.h \
+	IteratableChunkChecksumValidator.cc \
 	IteratableChunkChecksumValidator.h \
 	IteratableChecksumValidator.cc IteratableChecksumValidator.h \
 	CheckIntegrityCommand.cc CheckIntegrityCommand.h \
@@ -778,11 +779,12 @@ am__objects_14 = SocketCore.$(OBJEXT) Command.$(OBJEXT) \
 	ByteArrayDiskWriterFactory.$(OBJEXT) ServerHost.$(OBJEXT) \
 	HelpItem.$(OBJEXT) TaggedItem.$(OBJEXT) TagContainer.$(OBJEXT) \
 	HelpItemFactory.$(OBJEXT) SingleFileDownloadContext.$(OBJEXT) \
-	TimedHaltCommand.$(OBJEXT) $(am__objects_1) $(am__objects_2) \
+	TimedHaltCommand.$(OBJEXT) ProtocolDetector.$(OBJEXT) \
-	$(am__objects_3) $(am__objects_4) $(am__objects_5) \
+	$(am__objects_1) $(am__objects_2) $(am__objects_3) \
-	$(am__objects_6) $(am__objects_7) $(am__objects_8) \
+	$(am__objects_4) $(am__objects_5) $(am__objects_6) \
-	$(am__objects_9) $(am__objects_10) $(am__objects_11) \
+	$(am__objects_7) $(am__objects_8) $(am__objects_9) \
-	$(am__objects_12) $(am__objects_13)
+	$(am__objects_10) $(am__objects_11) $(am__objects_12) \
+	$(am__objects_13)
 am_libaria2c_a_OBJECTS = $(am__objects_14)
 libaria2c_a_OBJECTS = $(am_libaria2c_a_OBJECTS)
 am__installdirs = "$(DESTDIR)$(bindir)"
@@ -1116,7 +1118,8 @@ SRCS = Socket.h SocketCore.cc SocketCore.h BinaryStream.h Command.cc \
 	PostDownloadHandler.h PreDownloadHandler.h SingletonHolder.h \
 	TrueRequestGroupCriteria.h a2algo.h a2functional.h a2io.h \
 	a2netcompat.h a2time.h array_fun.h help_tags.h prefs.h \
-	usage_text.h $(am__append_1) $(am__append_2) $(am__append_3) \
+	usage_text.h ProtocolDetector.cc ProtocolDetector.h \
+	$(am__append_1) $(am__append_2) $(am__append_3) \
 	$(am__append_4) $(am__append_5) $(am__append_6) \
 	$(am__append_7) $(am__append_8) $(am__append_9) \
 	$(am__append_10) $(am__append_11) $(am__append_12) \
@@ -1431,6 +1434,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/PiecedSegment.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/PiecesMetalinkParserState.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Platform.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ProtocolDetector.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/RealtimeCommand.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ReceiverMSEHandshakeCommand.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Request.Po@am__quote@

src/ProtocolDetector.cc

@@ -0,0 +1,76 @@
+/* <!-- copyright */
+/*
+ * aria2 - The high speed download utility
+ *
+ * Copyright (C) 2006 Tatsuhiro Tsujikawa
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of portions of this program with the
+ * OpenSSL library under certain conditions as described in each
+ * individual source file, and distribute linked combinations
+ * including the two.
+ * You must obey the GNU General Public License in all respects
+ * for all of the code used other than OpenSSL.  If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so.  If you
+ * do not wish to do so, delete this exception statement from your
+ * version.  If you delete this exception statement from all source
+ * files in the program, then also delete it here.
+ */
+/* copyright --> */
+#include "ProtocolDetector.h"
+#include "Request.h"
+#include <cstring>
+#include <fstream>
+#include <iomanip>
+
+namespace aria2 {
+
+ProtocolDetector::ProtocolDetector() {}
+
+ProtocolDetector::~ProtocolDetector() {}
+
+bool ProtocolDetector::isStreamProtocol(const std::string& uri) const
+{
+  return Request().setUrl(uri);
+}
+
+bool ProtocolDetector::guessTorrentFile(const std::string& uri) const
+{
+  std::ifstream in(uri.c_str());
+  if(in) {
+    char head;
+    in >> head;
+    return head == 'd';
+  } else {
+    return false;
+  }
+}
+
+bool ProtocolDetector::guessMetalinkFile(const std::string& uri) const
+{
+  std::ifstream in(uri.c_str());
+  if(in) {
+    char head[6];
+    in >> std::setw(6) >> head;
+    return strcmp(head, "<?xml") == 0;
+  } else {
+    return false;
+  }
+}
+
+} // namespace aria2

src/ProtocolDetector.h

@@ -0,0 +1,63 @@
+/* <!-- copyright */
+/*
+ * aria2 - The high speed download utility
+ *
+ * Copyright (C) 2006 Tatsuhiro Tsujikawa
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of portions of this program with the
+ * OpenSSL library under certain conditions as described in each
+ * individual source file, and distribute linked combinations
+ * including the two.
+ * You must obey the GNU General Public License in all respects
+ * for all of the code used other than OpenSSL.  If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so.  If you
+ * do not wish to do so, delete this exception statement from your
+ * version.  If you delete this exception statement from all source
+ * files in the program, then also delete it here.
+ */
+/* copyright --> */
+#ifndef _D_PROTOCOL_DETECTOR_H_
+#define _D_PROTOCOL_DETECTOR_H_
+
+#include "common.h"
+#include <string>
+
+namespace aria2 {
+
+class ProtocolDetector {
+public:
+  ProtocolDetector();
+
+  ~ProtocolDetector();
+
+  // Returns true if uri is http(s)/ftp, otherwise returns false.
+  bool isStreamProtocol(const std::string& uri) const;
+
+  // Returns true if ProtocolDetector thinks uri is a path of BitTorrent
+  // metainfo file, otherwise returns false.
+  bool guessTorrentFile(const std::string& uri) const;
+
+  // Returns true if ProtocolDetector thinks uri is a path of Metalink XML
+  // file, otherwise return false.
+  bool guessMetalinkFile(const std::string& uri) const;
+};
+
+} // namespace aria2
+
+#endif // _D_PROTOCOL_DETECTOR_H_

src/main.cc

@@ -61,6 +61,7 @@
 #include "DefaultBtContext.h"
 #include "FileEntry.h"
 #include "RequestGroup.h"
+#include "ProtocolDetector.h"
 #ifdef ENABLE_METALINK
 # include "MetalinkHelper.h"
 # include "Metalink2RequestGroup.h"
@@ -74,6 +75,7 @@
 #include <unistd.h>
 #include <fstream>
 #include <iostream>
+#include <numeric>
 extern char* optarg;
 extern int optind, opterr, optopt;
 #include <getopt.h>
@@ -120,6 +122,24 @@ RequestGroupHandle createRequestGroup(const Option* op, const std::deque<std::st
 extern Option* option_processing(int argc, char* const argv[]);
 
 #ifdef ENABLE_BITTORRENT
+
+SharedHandle<RequestGroup>
+createBtRequestGroup(const std::string& torrentFilePath,
+		     Option* op,
+		     const std::deque<std::string>& auxUris)
+{
+  SharedHandle<RequestGroup> rg = new RequestGroup(op, auxUris);
+  SharedHandle<DefaultBtContext> btContext = new DefaultBtContext();
+  btContext->load(torrentFilePath);// may throw exception
+  if(op->defined(PREF_PEER_ID_PREFIX)) {
+    btContext->setPeerIdPrefix(op->get(PREF_PEER_ID_PREFIX));
+  }
+  btContext->setDir(op->get(PREF_DIR));
+  rg->setDownloadContext(btContext);
+  btContext->setOwnerRequestGroup(rg.get());
+  return rg;
+}
+
 int32_t downloadBitTorrent(Option* op, const std::deque<std::string>& uri)
 {
   std::deque<std::string> nargs;
@@ -132,15 +152,8 @@ int32_t downloadBitTorrent(Option* op, const std::deque<std::string>& uri)
   ncopy(nargs.begin(), nargs.end(), op->getAsInt(PREF_SPLIT),
 	std::back_inserter(xargs));
   
-  RequestGroupHandle rg = new RequestGroup(op, xargs);
+  RequestGroupHandle rg = createBtRequestGroup(op->get(PREF_TORRENT_FILE),
-  DefaultBtContextHandle btContext = new DefaultBtContext();
+					       op, xargs);
-  btContext->load(op->get(PREF_TORRENT_FILE));
-  if(op->defined(PREF_PEER_ID_PREFIX)) {
-    btContext->setPeerIdPrefix(op->get(PREF_PEER_ID_PREFIX));
-  }
-  btContext->setDir(op->get(PREF_DIR));
-  rg->setDownloadContext(btContext);
-  btContext->setOwnerRequestGroup(rg.get());
   
   RequestGroups groups;
   groups.push_back(rg);
@@ -159,6 +172,55 @@ int32_t downloadMetalink(Option* op)
 }
 #endif // ENABLE_METALINK
 
+class AccRequestGroup {
+private:
+  ProtocolDetector _detector;
+  Option* _op;
+public:
+  AccRequestGroup(Option* op):_op(op) {}
+
+  std::deque<SharedHandle<RequestGroup> >&
+  operator()(std::deque<SharedHandle<RequestGroup> >& groups,
+	     const std::string& uri)
+  {
+    if(_detector.isStreamProtocol(uri)) {
+      std::deque<std::string> xuris;
+      for(size_t count = _op->getAsInt(PREF_SPLIT); count; --count) {
+	xuris.push_back(uri);
+      }
+      RequestGroupHandle rg = createRequestGroup(_op, xuris);
+      groups.push_back(rg);
+    }
+#ifdef ENABLE_BITTORRENT
+    else if(_detector.guessTorrentFile(uri)) {
+      try {
+	groups.push_back(createBtRequestGroup(uri, _op,
+					      std::deque<std::string>()));
+      } catch(RecoverableException* e) {
+	// error occurred while parsing torrent file.
+	// We simply ignore it.	
+      }
+    }
+#endif // ENABLE_BITTORRENT
+#ifdef ENABLE_METALINK
+    else if(_detector.guessMetalinkFile(uri)) {
+      try {
+	std::deque<SharedHandle<RequestGroup> > metalinkGroups =
+	  Metalink2RequestGroup(_op).generate(uri);
+	groups.insert(groups.end(), metalinkGroups.begin(), metalinkGroups.end());
+      } catch(RecoverableException* e) {
+	// error occurred while parsing metalink file.
+	// We simply ignore it.
+      }
+    }
+#endif // ENABLE_METALINK
+    else {
+      LogFactory::getInstance()->error(MSG_UNRECOGNIZED_URI, (uri).c_str());
+    }
+    return groups;
+  }
+};
+
 int32_t downloadUriList(Option* op, std::istream& in)
 {
   UriListParser p;
@@ -167,13 +229,17 @@ int32_t downloadUriList(Option* op, std::istream& in)
     std::deque<std::string> uris = p.parseNext(in);
     if(uris.size() == 1 && op->get(PREF_PARAMETERIZED_URI) == V_TRUE) {
       std::deque<std::string> unfoldedURIs = unfoldURI(uris);
-      for(std::deque<std::string>::const_iterator itr = unfoldedURIs.begin();
+      std::deque<SharedHandle<RequestGroup> > thisGroups =
-	  itr != unfoldedURIs.end(); ++itr) {
+	std::accumulate(unfoldedURIs.begin(), unfoldedURIs.end(),
-	std::deque<std::string> xuris;
+			std::deque<SharedHandle<RequestGroup> >(),
-	ncopy(itr, itr+1, op->getAsInt(PREF_SPLIT), std::back_inserter(xuris));
+			AccRequestGroup(op));
-	SharedHandle<RequestGroup> rg = createRequestGroup(op, xuris);
+      groups.insert(groups.end(), thisGroups.begin(), thisGroups.end());
-	groups.push_back(rg);
+    } else if(uris.size() == 1) {
-      }
+      std::deque<SharedHandle<RequestGroup> > thisGroups =
+	std::accumulate(uris.begin(), uris.end(),
+			std::deque<SharedHandle<RequestGroup> >(),
+			AccRequestGroup(op));
+      groups.insert(groups.end(), thisGroups.begin(), thisGroups.end());
     } else if(uris.size() > 0) {
       std::deque<std::string> xuris;
       ncopy(uris.begin(), uris.end(), op->getAsInt(PREF_SPLIT),
@@ -208,14 +274,9 @@ int32_t downloadUri(Option* op, const std::deque<std::string>& uris)
   }
   RequestGroups groups;
   if(op->get(PREF_FORCE_SEQUENTIAL) == V_TRUE) {
-    for(std::deque<std::string>::const_iterator itr = nargs.begin();
+    groups = std::accumulate(nargs.begin(), nargs.end(),
-	itr != nargs.end(); ++itr) {
+			     std::deque<SharedHandle<RequestGroup> >(),
-      std::deque<std::string> xuris;
+			     AccRequestGroup(op));
-      ncopy(itr, itr+1, op->getAsInt(PREF_SPLIT),
-	    std::back_inserter(xuris));
-      RequestGroupHandle rg = createRequestGroup(op, xuris);
-      groups.push_back(rg);
-    }
   } else {
     std::deque<std::string> xargs;
     ncopy(nargs.begin(), nargs.end(), op->getAsInt(PREF_SPLIT),

test/ProtocolDetectorTest.cc

@@ -0,0 +1,53 @@
+#include "ProtocolDetector.h"
+#include "Exception.h"
+#include "Util.h"
+#include <iostream>
+#include <cppunit/extensions/HelperMacros.h>
+
+namespace aria2 {
+
+class ProtocolDetectorTest:public CppUnit::TestFixture {
+
+  CPPUNIT_TEST_SUITE(ProtocolDetectorTest);
+  CPPUNIT_TEST(testIsStreamProtocol);
+  CPPUNIT_TEST(testGuessTorrentFile);
+  CPPUNIT_TEST(testGuessMetalinkFile);
+  CPPUNIT_TEST_SUITE_END();
+public:
+  void setUp() {}
+
+  void tearDown() {}
+
+  void testIsStreamProtocol();
+  void testGuessTorrentFile();
+  void testGuessMetalinkFile();
+};
+
+
+CPPUNIT_TEST_SUITE_REGISTRATION(ProtocolDetectorTest);
+
+void ProtocolDetectorTest::testIsStreamProtocol()
+{
+  ProtocolDetector detector;
+  CPPUNIT_ASSERT(detector.isStreamProtocol("http://localhost/index.html"));
+  CPPUNIT_ASSERT(detector.isStreamProtocol("ftp://localhost/index.html"));
+  CPPUNIT_ASSERT(!detector.isStreamProtocol("/home/web/localhost/index.html"));
+}
+
+void ProtocolDetectorTest::testGuessTorrentFile()
+{
+  ProtocolDetector detector;
+  CPPUNIT_ASSERT(detector.guessTorrentFile("test.torrent"));
+  CPPUNIT_ASSERT(!detector.guessTorrentFile("http://localhost/test.torrent"));
+  CPPUNIT_ASSERT(!detector.guessTorrentFile("test.xml"));
+}
+
+void ProtocolDetectorTest::testGuessMetalinkFile()
+{
+  ProtocolDetector detector;
+  CPPUNIT_ASSERT(detector.guessMetalinkFile("test.xml"));
+  CPPUNIT_ASSERT(!detector.guessMetalinkFile("http://localhost/test.xml"));
+  CPPUNIT_ASSERT(!detector.guessMetalinkFile("test.torrent"));
+}
+
+} // namespace aria2