Support relative URI in Metalink file.

If relative URI is found in Metalink file, aria2 resolves its full URI
contatenating the URI from which Metalink file is retrieved and
relative URI in Metalink file. This feature is not available if
Metalink file in local disk is specified in command line.

src/Metalink2RequestGroup.cc

@@ -110,10 +110,11 @@ void
 Metalink2RequestGroup::generate
 (std::vector<SharedHandle<RequestGroup> >& groups,
  const std::string& metalinkFile,
- const SharedHandle<Option>& option)
+ const SharedHandle<Option>& option,
+ const std::string& baseUri)
 {
   std::vector<SharedHandle<MetalinkEntry> > entries;
-  metalink::parseAndQuery(entries, metalinkFile, option.get());
+  metalink::parseAndQuery(entries, metalinkFile, option.get(), baseUri);
   std::vector<SharedHandle<RequestGroup> > tempgroups;
   createRequestGroup(tempgroups, entries, option);
   SharedHandle<MetadataInfo> mi;
@@ -130,10 +131,11 @@ void
 Metalink2RequestGroup::generate
 (std::vector<SharedHandle<RequestGroup> >& groups,
  const SharedHandle<BinaryStream>& binaryStream,
- const SharedHandle<Option>& option)
+ const SharedHandle<Option>& option,
+ const std::string& baseUri)
 {
   std::vector<SharedHandle<MetalinkEntry> > entries;
-  metalink::parseAndQuery(entries, binaryStream, option.get());
+  metalink::parseAndQuery(entries, binaryStream, option.get(), baseUri);
   std::vector<SharedHandle<RequestGroup> > tempgroups;
   createRequestGroup(tempgroups, entries, option);
   SharedHandle<MetadataInfo> mi(new MetadataInfo());

src/Metalink2RequestGroup.h

@@ -36,10 +36,12 @@
 #define D_METALINK_2_REQUEST_GROUP_H
 
 #include "common.h"
-#include "SharedHandle.h"
 #include <string>
 #include <vector>
 
+#include "SharedHandle.h"
+#include "A2STR.h"
+
 namespace aria2 {
 
 class Option;
@@ -58,11 +60,13 @@ public:
 
   void generate(std::vector<SharedHandle<RequestGroup> >& groups,
                 const std::string& metalinkFile,
-                const SharedHandle<Option>& option);
+                const SharedHandle<Option>& option,
+                const std::string& baseUri = A2STR::NIL);
 
   void generate(std::vector<SharedHandle<RequestGroup> >& groups,
                 const SharedHandle<BinaryStream>& binaryStream,
-                const SharedHandle<Option>& option);
+                const SharedHandle<Option>& option,
+                const std::string& baseUri = A2STR::NIL);
 };
 
 } // namespace aria2

src/MetalinkParserController.cc

@@ -43,13 +43,17 @@
 #include "FileEntry.h"
 #include "a2functional.h"
 #include "A2STR.h"
+#include "uri.h"
+#include "Signature.h"
+#include "util.h"
 #ifdef ENABLE_MESSAGE_DIGEST
 # include "Checksum.h"
 # include "ChunkChecksum.h"
 # include "MessageDigest.h"
 #endif // ENABLE_MESSAGE_DIGEST
-#include "Signature.h"
+#ifdef ENABLE_BITTORRENT
-#include "util.h"
+# include "magnet.h"
+#endif // ENABLE_BITTORRENT
 
 namespace aria2 {
 
@@ -166,14 +170,15 @@ void MetalinkParserController::setURLOfResource(const std::string& url)
   if(!tResource_) {
     return;
   }
-  tResource_->url = url;
+  std::string u = uri::joinUri(baseUri_, url);
-  // Metalink4Spec
+  uri::UriStruct us;
+  if(uri::parse(us, u)) {
+    tResource_->url = u;
     if(tResource_->type == MetalinkResource::TYPE_UNKNOWN) {
-    // guess from URI sheme
+      setTypeOfResource(us.protocol);
-    std::string::size_type pos = url.find("://");
-    if(pos != std::string::npos) {
-      setTypeOfResource(url.substr(0, pos));
     }
+  } else {
+    tResource_->url = url;
   }
 }
 
@@ -563,7 +568,20 @@ void MetalinkParserController::setURLOfMetaurl(const std::string& url)
   if(!tMetaurl_) {
     return;
   }
+#ifdef ENABLE_BITTORRENT
+  if(magnet::parse(url)) {
     tMetaurl_->url = url;
+  } else
+#endif // ENABLE_BITTORRENT
+    {
+      std::string u = uri::joinUri(baseUri_, url);
+      uri::UriStruct us;
+      if(uri::parse(us, u)) {
+        tMetaurl_->url = u;
+      } else {
+        tMetaurl_->url = url;
+      }
+    }
 }
 
 void MetalinkParserController::setMediatypeOfMetaurl

src/MetalinkParserController.h

@@ -81,6 +81,7 @@ private:
 #endif // ENABLE_MESSAGE_DIGEST
 
   SharedHandle<Signature> tSignature_;
+  std::string baseUri_;
 public:
   MetalinkParserController();
 
@@ -190,6 +191,11 @@ public:
   void commitMetaurlTransaction();
 
   void cancelMetaurlTransaction();
+
+  void setBaseUri(const std::string& baseUri)
+  {
+    baseUri_ = baseUri;
+  }
 };
 
 } // namespace aria2

src/MetalinkParserStateMachine.cc

@@ -541,5 +541,9 @@ std::string MetalinkParserStateMachine::getErrorString() const
   return error.str();
 }
 
+void MetalinkParserStateMachine::setBaseUri(const std::string& uri)
+{
+  ctrl_->setBaseUri(uri);
+}
 
 } // namespace aria2

src/MetalinkParserStateMachine.h

@@ -266,6 +266,8 @@ public:
   {
     return ctrl_->getResult();
   }
+
+  void setBaseUri(const std::string& uri);
 };
 
 } //  namespace aria2

src/MetalinkPostDownloadHandler.cc

@@ -33,6 +33,9 @@
  */
 /* copyright --> */
 #include "MetalinkPostDownloadHandler.h"
+
+#include <deque>
+
 #include "RequestGroup.h"
 #include "Metalink2RequestGroup.h"
 #include "Logger.h"
@@ -47,6 +50,7 @@
 #include "DownloadContext.h"
 #include "download_helper.h"
 #include "fmt.h"
+#include "FileEntry.h"
 
 namespace aria2 {
 
@@ -63,6 +67,31 @@ MetalinkPostDownloadHandler::MetalinkPostDownloadHandler()
 
 MetalinkPostDownloadHandler::~MetalinkPostDownloadHandler() {}
 
+namespace {
+const std::string& getBaseUri(RequestGroup* requestGroup)
+{
+  const SharedHandle<DownloadContext>& dctx =
+    requestGroup->getDownloadContext();
+  if(dctx->getFileEntries().empty()) {
+    return A2STR::NIL;
+  } else {
+    // TODO Check download result for each URI
+    const SharedHandle<FileEntry>& entry = dctx->getFirstFileEntry();
+    const std::deque<std::string>& spentUris = entry->getSpentUris();
+    if(spentUris.empty()) {
+      const std::deque<std::string>& remainingUris = entry->getRemainingUris();
+      if(remainingUris.empty()) {
+        return A2STR::NIL;
+      } else {
+        return remainingUris.front();
+      }
+    } else {
+      return spentUris.back();
+    }
+  }
+}
+} // namespace
+
 void MetalinkPostDownloadHandler::getNextRequestGroups
 (std::vector<SharedHandle<RequestGroup> >& groups,
  RequestGroup* requestGroup)
@@ -74,9 +103,10 @@ void MetalinkPostDownloadHandler::getNextRequestGroups
   try {
     diskAdaptor->openExistingFile();
     //requestOption.put(PREF_DIR, requestGroup->getDownloadContext()->getDir());
+    const std::string& baseUri = getBaseUri(requestGroup);
     std::vector<SharedHandle<RequestGroup> > newRgs;
     Metalink2RequestGroup().generate(newRgs, diskAdaptor,
-                                     requestGroup->getOption());
+                                     requestGroup->getOption(), baseUri);
     requestGroup->followedBy(newRgs.begin(), newRgs.end());
     SharedHandle<MetadataInfo> mi =
       createMetadataInfoFromFirstFileEntry(requestGroup->getDownloadContext());

src/XML2SAXMetalinkProcessor.cc

@@ -185,9 +185,12 @@ MetalinkProcessor::MetalinkProcessor() {}
 MetalinkProcessor::~MetalinkProcessor() {}
 
 SharedHandle<Metalinker>
-MetalinkProcessor::parseFile(const std::string& filename)
+MetalinkProcessor::parseFile
+(const std::string& filename,
+ const std::string& baseUri)
 {
   stm_.reset(new MetalinkParserStateMachine());
+  stm_->setBaseUri(baseUri);
   SharedHandle<SessionData> sessionData(new SessionData(stm_));
   // Old libxml2(at least 2.7.6, Ubuntu 10.04LTS) does not read stdin
   // when "/dev/stdin" is passed as filename while 2.7.7 does. So we
@@ -216,9 +219,12 @@ MetalinkProcessor::parseFile(const std::string& filename)
 }
          
 SharedHandle<Metalinker>
-MetalinkProcessor::parseFromBinaryStream(const SharedHandle<BinaryStream>& binaryStream)
+MetalinkProcessor::parseFromBinaryStream
+(const SharedHandle<BinaryStream>& binaryStream,
+ const std::string& baseUri)
 {
   stm_.reset(new MetalinkParserStateMachine());
+  stm_->setBaseUri(baseUri);
   size_t bufSize = 4096;
   unsigned char buf[bufSize];
 

src/XML2SAXMetalinkProcessor.h

@@ -43,6 +43,7 @@
 #include <libxml/xpath.h>
 
 #include "SharedHandle.h"
+#include "A2STR.h"
 
 namespace aria2 {
 
@@ -58,10 +59,13 @@ public:
 
   ~MetalinkProcessor();
 
-  SharedHandle<Metalinker> parseFile(const std::string& filename);
+  SharedHandle<Metalinker> parseFile
+  (const std::string& filename,
+   const std::string& baseUri = A2STR::NIL);
 
   SharedHandle<Metalinker> parseFromBinaryStream
-  (const SharedHandle<BinaryStream>& binaryStream);
+  (const SharedHandle<BinaryStream>& binaryStream,
+   const std::string& baseUri = A2STR::NIL);
 };
 
 } // namespace aria2

src/metalink_helper.cc

@@ -65,20 +65,23 @@ void query
 void parseAndQuery
 (std::vector<SharedHandle<MetalinkEntry> >& result,
  const std::string& filename,
- const Option* option)
+ const Option* option,
+ const std::string& baseUri)
 {
   MetalinkProcessor proc;
-  SharedHandle<Metalinker> metalinker = proc.parseFile(filename);
+  SharedHandle<Metalinker> metalinker = proc.parseFile(filename, baseUri);
   query(result, metalinker, option);
 }
 
 void parseAndQuery
 (std::vector<SharedHandle<MetalinkEntry> >& result,
  const SharedHandle<BinaryStream>& binaryStream,
- const Option* option)
+ const Option* option,
+ const std::string& baseUri)
 {
   MetalinkProcessor proc;
-  SharedHandle<Metalinker> metalinker =proc.parseFromBinaryStream(binaryStream);
+  SharedHandle<Metalinker> metalinker =
+    proc.parseFromBinaryStream(binaryStream, baseUri);
   query(result, metalinker, option);
 }
 

src/metalink_helper.h

@@ -41,6 +41,7 @@
 #include <vector>
 
 #include "SharedHandle.h"
+#include "A2STR.h"
 
 namespace aria2 {
 
@@ -54,12 +55,14 @@ namespace metalink {
 void parseAndQuery
 (std::vector<SharedHandle<MetalinkEntry> >& result,
  const std::string& filename,
- const Option* option);
+ const Option* option,
+ const std::string& baseUri = A2STR::NIL);
 
 void parseAndQuery
 (std::vector<SharedHandle<MetalinkEntry> >& result,
  const SharedHandle<BinaryStream>& binaryStream,
- const Option* option);
+ const Option* option,
+ const std::string& baseUri = A2STR::NIL);
 
 void groupEntryByMetaurlName
 (std::vector<

test/MetalinkParserControllerTest.cc

@@ -20,6 +20,7 @@ class MetalinkParserControllerTest:public CppUnit::TestFixture {
   CPPUNIT_TEST_SUITE(MetalinkParserControllerTest);
   CPPUNIT_TEST(testEntryTransaction);
   CPPUNIT_TEST(testResourceTransaction);
+  CPPUNIT_TEST(testResourceTransaction_withBaseUri);
   CPPUNIT_TEST(testMetaurlTransaction);
 #ifdef ENABLE_MESSAGE_DIGEST
   CPPUNIT_TEST(testChecksumTransaction);
@@ -38,6 +39,7 @@ public:
 
   void testEntryTransaction();
   void testResourceTransaction();
+  void testResourceTransaction_withBaseUri();
   void testMetaurlTransaction();
 #ifdef ENABLE_MESSAGE_DIGEST
   void testChecksumTransaction();
@@ -110,6 +112,46 @@ void MetalinkParserControllerTest::testResourceTransaction()
   }
 }
 
+void MetalinkParserControllerTest::testResourceTransaction_withBaseUri()
+{
+  MetalinkParserController ctrl;
+  ctrl.setBaseUri("http://base/dir/file");
+  ctrl.newEntryTransaction();
+  ctrl.newResourceTransaction();
+  ctrl.setURLOfResource("aria2.tar.bz2");
+  ctrl.commitResourceTransaction();
+#ifdef ENABLE_BITTORRENT
+  ctrl.newMetaurlTransaction();
+  ctrl.setURLOfMetaurl("/meta/aria2.tar.bz2.torrent");
+  ctrl.setMediatypeOfMetaurl("torrent");
+  ctrl.commitMetaurlTransaction();
+  ctrl.newMetaurlTransaction();
+  ctrl.setURLOfMetaurl("magnet:?xt=urn:btih:248d0a1cd08284299de78d5c1ed359bb46717d8c");
+  ctrl.setMediatypeOfMetaurl("torrent");
+  ctrl.commitMetaurlTransaction();
+#endif // ENABLE_BITTORRENT
+  ctrl.commitEntryTransaction();
+  {
+    SharedHandle<Metalinker> m = ctrl.getResult();
+    CPPUNIT_ASSERT_EQUAL((size_t)1, m->getEntries()[0]->resources.size());
+    SharedHandle<MetalinkResource> res = m->getEntries()[0]->resources[0];
+    CPPUNIT_ASSERT_EQUAL(std::string("http://base/dir/aria2.tar.bz2"),
+                         res->url);
+    CPPUNIT_ASSERT_EQUAL(MetalinkResource::TYPE_HTTP, res->type);
+
+#ifdef ENABLE_BITTORRENT
+    CPPUNIT_ASSERT_EQUAL((size_t)2, m->getEntries()[0]->metaurls.size());
+    SharedHandle<MetalinkMetaurl> metaurl = m->getEntries()[0]->metaurls[0];
+    CPPUNIT_ASSERT_EQUAL(std::string("http://base/meta/aria2.tar.bz2.torrent"),
+                         metaurl->url);
+
+    metaurl = m->getEntries()[0]->metaurls[1];
+    CPPUNIT_ASSERT_EQUAL(std::string("magnet:?xt=urn:btih:248d0a1cd08284299de78d5c1ed359bb46717d8c"),
+                         metaurl->url);
+#endif // ENABLE_BITTORRENT
+  }
+}
+
 void MetalinkParserControllerTest::testMetaurlTransaction()
 {
   MetalinkParserController ctrl;

test/MetalinkPostDownloadHandlerTest.cc

@@ -17,6 +17,7 @@ class MetalinkPostDownloadHandlerTest:public CppUnit::TestFixture {
   CPPUNIT_TEST(testCanHandle_extension);
   CPPUNIT_TEST(testCanHandle_contentType);
   CPPUNIT_TEST(testGetNextRequestGroups);
+  CPPUNIT_TEST(testGetNextRequestGroups_withBaseUri);
   CPPUNIT_TEST_SUITE_END();
 private:
   SharedHandle<Option> option_;
@@ -29,6 +30,7 @@ public:
   void testCanHandle_extension();
   void testCanHandle_contentType();
   void testGetNextRequestGroups();
+  void testGetNextRequestGroups_withBaseUri();
 };
 
 
@@ -83,4 +85,23 @@ void MetalinkPostDownloadHandlerTest::testGetNextRequestGroups()
 #endif // ENABLE_BITTORRENT
 }
 
+void MetalinkPostDownloadHandlerTest::testGetNextRequestGroups_withBaseUri()
+{
+  SharedHandle<DownloadContext> dctx
+    (new DownloadContext(1024, 0, A2_TEST_DIR"/base_uri.xml"));
+  dctx->getFirstFileEntry()->addUri("http://base/dir/base_uri.xml");
+  RequestGroup rg(option_);
+  rg.setDownloadContext(dctx);
+  rg.initPieceStorage();
+  rg.getPieceStorage()->getDiskAdaptor()->enableReadOnly();
+
+  MetalinkPostDownloadHandler handler;
+  std::vector<SharedHandle<RequestGroup> > groups;
+  handler.getNextRequestGroups(groups, &rg);
+  CPPUNIT_ASSERT_EQUAL((size_t)1, groups.size());
+  CPPUNIT_ASSERT_EQUAL(std::string("http://base/dir/example.ext"),
+                       groups[0]->getDownloadContext()->
+                       getFirstFileEntry()->getRemainingUris()[0]);
+}
+
 } // namespace aria2

test/base_uri.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<metalink xmlns="urn:ietf:params:xml:ns:metalink">
+  <file name="example.ext">
+    <url>example.ext</url>
+  </file>
+</metalink>