* Added HTTPS support.

* Added SocketCore. Socket is now handle class for SocketCore.
	* Fixed bug in ChunkedEncoding: expanding buffer size is wrong
	* Fixed bug in DownloadCommand: In Chunked Encoding, it wrongly
	adds to Segment.ds buff length from the socket.

ChangeLog

@@ -1,3 +1,12 @@
+2006-02-18  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
+
+	* Added HTTPS support.
+	* Added SocketCore. Socket is now handle class for SocketCore.
+	* Fixed bug in ChunkedEncoding: expanding buffer size is wrong
+	* Fixed bug in DownloadCommand: In Chunked Encoding, it wrongly
+	adds to Segment.ds buff length from the socket.
+
 2006-02-17  Tatsuhiro Tsujikawa <tsujikawa at rednoah dot com>
+	
 	* Release 0.1.0
 	

TODO

@@ -1,4 +1,7 @@
 * Add HTTP POST support
-* Add HTTPS support using OpenSSL
 * Add expires handling for Cookie
+* Add Referer support
+* Fix no wait retry in HttpInitiateConnectionCommand, HttpRequestCommand, HttpResponseCommand, except for redirection.
 * Add FTP support
+* Add SSL server cert verification
+* Add SSL client cert support

config.h.in

@@ -3,6 +3,9 @@
 /* Define to 1 if you have the <arpa/inet.h> header file. */
 #undef HAVE_ARPA_INET_H
 
+/* Define to 1 if you don't have `vprintf' but do have `_doprnt.' */
+#undef HAVE_DOPRNT
+
 /* Define to 1 if you have the <fcntl.h> header file. */
 #undef HAVE_FCNTL_H
 
@@ -18,6 +21,9 @@
 /* Define to 1 if you have the <inttypes.h> header file. */
 #undef HAVE_INTTYPES_H
 
+/* Define to 1 if you have the `ssl' library (-lssl). */
+#undef HAVE_LIBSSL
+
 /* Define to 1 if you have the <memory.h> header file. */
 #undef HAVE_MEMORY_H
 
@@ -91,6 +97,9 @@
 /* Define to 1 if you have the <unistd.h> header file. */
 #undef HAVE_UNISTD_H
 
+/* Define to 1 if you have the `vprintf' function. */
+#undef HAVE_VPRINTF
+
 /* Define to 1 if the system has the type `_Bool'. */
 #undef HAVE__BOOL
 

configure

@@ -5336,6 +5336,80 @@ fi
 done
 
 
+
+echo "$as_me:$LINENO: checking for SSL_library_init in -lssl" >&5
+echo $ECHO_N "checking for SSL_library_init in -lssl... $ECHO_C" >&6
+if test "${ac_cv_lib_ssl_SSL_library_init+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lssl  $LIBS"
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char SSL_library_init ();
+int
+main ()
+{
+SSL_library_init ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_lib_ssl_SSL_library_init=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_lib_ssl_SSL_library_init=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_lib_ssl_SSL_library_init" >&5
+echo "${ECHO_T}$ac_cv_lib_ssl_SSL_library_init" >&6
+if test $ac_cv_lib_ssl_SSL_library_init = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBSSL 1
+_ACEOF
+
+  LIBS="-lssl $LIBS"
+
+fi
+
+
                               ac_config_files="$ac_config_files Makefile src/Makefile test/Makefile"
 
 cat >confcache <<\_ACEOF

configure.in

@@ -31,5 +31,7 @@ AC_FUNC_STAT
 AC_FUNC_VPRINTF
 AC_CHECK_FUNCS([getpagesize gethostbyname gettimeofday memset mkdir rmdir select socket strcasecmp strerror strstr strtol])
 
+AC_CHECK_LIB([ssl], [SSL_library_init])
+
 AC_CONFIG_FILES([Makefile src/Makefile test/Makefile])
 AC_OUTPUT

src/AbstractCommand.h

@@ -26,6 +26,7 @@
 #include "Request.h"
 #include "DownloadEngine.h"
 #include "SegmentMan.h"
+#include "common.h"
 
 class AbstractCommand : public Command {
 private:

src/ChunkedEncoding.cc

@@ -25,7 +25,7 @@
 #include <string.h>
 #include <errno.h>
 
-#define MAX_BUFSIZE 8192
+#define MAX_BUFSIZE (1024*1024)
 
 ChunkedEncoding::ChunkedEncoding() {
   strbufSize = 4096;
@@ -161,11 +161,11 @@ void ChunkedEncoding::addBuffer(const char* inbuf, int inlen) {
     if(strlen(strbuf)+inlen+1 > MAX_BUFSIZE) {
       throw new DlAbortEx(EX_TOO_LARGE_CHUNK, strlen(strbuf)+inlen+1);
     }
-    char* temp = new char[strlen(strbuf)+inlen+1];
+    strbufSize = strlen(strbuf)+inlen+1;
+    char* temp = new char[strbufSize];
     memcpy(temp, strbuf, strlen(strbuf)+1);
     delete [] strbuf;
     strbuf = temp;
-    strbufSize = strlen(strbuf);
   }
   int origlen = strlen(strbuf);
   memcpy(strbuf+origlen, inbuf, inlen);

src/DownloadCommand.cc

@@ -48,10 +48,11 @@ bool DownloadCommand::executeInternal(Segment seg) {
     char infbuf[infbufSize];
     te->inflate(infbuf, infbufSize, buf, bufSize);
     e->diskWriter->writeData(infbuf, infbufSize, seg.sp+seg.ds);
+    seg.ds += infbufSize;
   } else {
     e->diskWriter->writeData(buf, bufSize, seg.sp+seg.ds);
-  }
     seg.ds += bufSize;
+  }
   
   if(te != NULL && te->finished()
      || te == NULL && seg.ds >= seg.ep-seg.sp+1

src/HttpConnection.cc

@@ -48,11 +48,14 @@ void HttpConnection::sendProxyRequest(const Request* req) {
 }
 
 string HttpConnection::getHost(const string& host, int port) {
-  return host+(port == 80 ? "" : ":"+Util::llitos(port));
+  return host+(port == 80 || port == 443 ? "" : ":"+Util::llitos(port));
 }
 
 string HttpConnection::createRequest(const Request* req, const Segment& segment) {
-  string request = string("GET ")+req->getCurrentUrl()+string(" HTTP/1.1\r\n")+
+  string request = string("GET ")+
+    req->getCurrentUrl()+
+    //(req->getDir() == "/" ? "/" : req->getDir()+"/")+req->getFile()+
+    string(" HTTP/1.1\r\n")+
     "Referer: \r\n"+
     "User-Agent: aria2\r\n"+
     "Connection: close\r\n"+

src/HttpRequestCommand.cc

@@ -36,6 +36,12 @@ HttpRequestCommand::~HttpRequestCommand() {}
 
 bool HttpRequestCommand::executeInternal(Segment seg) {
   socket->setNonBlockingMode();
+#ifdef HAVE_LIBSSL
+  // for SSL
+  if(req->getProtocol() == "https") {
+    socket->initiateSecureConnection();
+  }
+#endif // HAVE_LIBSSL
   HttpConnection httpConnection(cuid, socket, e->option, e->logger);
   // set seg to request in order to remember the request range
   req->seg = seg;

src/HttpResponseCommand.cc

@@ -133,6 +133,7 @@ void HttpResponseCommand::createHttpDownloadCommand(string transferEncoding) {
     throw new DlAbortEx(EX_TRANSFER_ENCODING_NOT_SUPPORTED, transferEncoding.c_str());
   } else {
     if(enc != NULL) {
+      command->transferEncoding = transferEncoding;
       enc->init();
     }
     e->commands.push(command);

src/InitiateConnectionCommandFactory.cc

@@ -23,7 +23,12 @@
 #include "HttpInitiateConnectionCommand.h"
 
 Command* InitiateConnectionCommandFactory::createInitiateConnectionCommand(int cuid, Request* req, DownloadEngine* e) {
-  if(req->getProtocol() == "http") {
+  if(req->getProtocol() == "http"
+#ifdef HAVE_LIBSSL
+     // for SSL
+     || req->getProtocol() == "https"
+#endif // HAVE_LIBSSL
+     ) {
     return new HttpInitiateConnectionCommand(cuid, req, e);
   } else {
     // these protocols are not supported yet

src/InitiateConnectionCommandFactory.h

@@ -24,6 +24,7 @@
 
 #include "Request.h"
 #include "DownloadEngine.h"
+#include "common.h"
 
 class InitiateConnectionCommandFactory {
 public:

src/Request.cc

@@ -26,6 +26,10 @@
 
 Request::Request():port(0), retryCount(0) {
   defaultPorts["http"] = 80;
+#ifdef HAVE_LIBSSL
+  // for SSL
+  defaultPorts["https"] = 443;
+#endif // HAVE_LIBSSL
   seg.sp = 0;
   seg.ep = 0;
   seg.ds = 0;

src/Request.h

@@ -25,6 +25,7 @@
 #include <map>
 #include <Segment.h>
 #include "CookieBox.h"
+#include "common.h"
 
 using namespace std;
 

src/Socket.cc

@@ -80,3 +80,10 @@ void Socket::readData(char* data, int& len, int timeout) {
 void Socket::peekData(char* data, int& len, int timeout) {
   core->peekData(data, len, timeout);
 }
+
+#ifdef HAVE_LIBSSL
+// for SSL
+void Socket::initiateSecureConnection() {
+  core->initiateSecureConnection();
+}
+#endif // HAVE_LIBSSL

src/Socket.h

@@ -24,6 +24,7 @@
 
 #include <string>
 #include "SocketCore.h"
+#include "common.h"
 
 using namespace std;
 
@@ -74,6 +75,11 @@ public:
   // Reads up to len bytes from this socket, but bytes are not removed from
   // this socket.
   void peekData(char* data, int& len, int timeout = 5);
+
+#ifdef HAVE_LIBSSL
+  // for SSL
+  void initiateSecureConnection();
+#endif // HAVE_LIBSSL
 };
 
 #endif // _D_SOCKET_H_

src/SocketCore.cc

@@ -32,8 +32,14 @@
 #include "DlRetryEx.h"
 #include "DlAbortEx.h"
 #include <errno.h>
+#include "message.h"
 
-SocketCore::SocketCore():sockfd(-1), use(1) {}
+SocketCore::SocketCore():sockfd(-1), use(1), secure(false)
+#ifdef HAVE_LIBSSL
+  // for SSL
+			 , sslCtx(NULL), ssl(NULL)
+#endif // HAVE_LIBSSL
+{}
 
 SocketCore::~SocketCore() {
   closeConnection();
@@ -87,10 +93,25 @@ void SocketCore::setNonBlockingMode() {
 }
 
 void SocketCore::closeConnection() {
+#ifdef HAVE_LIBSSL
+  // for SSL
+  if(secure) {
+    SSL_shutdown(ssl);
+  }
+#endif // HAVE_LIBSSL
   if(sockfd != -1) {
     close(sockfd);
     sockfd = -1;
   }
+#ifdef HAVE_LIBSSL
+  // for SSL
+  if(secure) {
+    SSL_free(ssl);
+    SSL_CTX_free(sslCtx);
+    ssl = NULL;
+    sslCtx = NULL;
+  }
+#endif // HAVE_LIBSSL
 }
 
 bool SocketCore::isWritable(int timeout) {
@@ -134,19 +155,65 @@ bool SocketCore::isReadable(int timeout) {
 }
 
 void SocketCore::writeData(const char* data, int len, int timeout) {
-  if(!isWritable(timeout) || send(sockfd, data, (size_t)len, 0) != len) {
+  if(!isWritable(timeout) ||
+     !secure && send(sockfd, data, (size_t)len, 0) != len
+#ifdef HAVE_LIBSSL
+     // for SSL
+     // TODO handling len == 0 case required
+     || secure && SSL_write(ssl, data, len) != len
+#endif // HAVE_LIBSSL
+     ) {
     throw new DlRetryEx(strerror(errno));
   }
 }
 
 void SocketCore::readData(char* data, int& len, int timeout) {
-  if(!isReadable(timeout) || (len = recv(sockfd, data, (size_t)len, 0)) < 0) {
+  if(!isReadable(timeout) ||
+     !secure && (len = recv(sockfd, data, (size_t)len, 0)) < 0
+#ifdef HAVE_LIBSSL
+     // for SSL
+     // TODO handling len == 0 case required
+     || secure && (len = SSL_read(ssl, data, len)) < 0
+#endif // HAVE_LIBSSL
+     ) {
     throw new DlRetryEx(strerror(errno));
   }
 }
 
 void SocketCore::peekData(char* data, int& len, int timeout) {
-  if(!isReadable(timeout) || (len = recv(sockfd, data, (size_t)len, MSG_PEEK)) < 0) {
+  if(!isReadable(timeout) ||
+     !secure && (len = recv(sockfd, data, (size_t)len, MSG_PEEK)) < 0
+#ifdef HAVE_LIBSSL
+     // for SSL
+     // TODO handling len == 0 case required
+     || secure && (len == SSL_peek(ssl, data, len)) < 0
+#endif // HAVE_LIBSSL
+     ) {
     throw new DlRetryEx(strerror(errno));
   }
 }
+
+#ifdef HAVE_LIBSSL
+// for SSL
+void SocketCore::initiateSecureConnection() {
+  if(!secure) {
+    sslCtx = SSL_CTX_new(SSLv23_client_method());
+    if(sslCtx == NULL) {
+      throw new DlAbortEx(EX_SSL_INIT_FAILURE);
+    }
+    SSL_CTX_set_mode(sslCtx, SSL_MODE_AUTO_RETRY);
+    ssl = SSL_new(sslCtx);
+    if(ssl == NULL) {
+      throw new DlAbortEx(EX_SSL_INIT_FAILURE);
+    }
+    if(SSL_set_fd(ssl, sockfd) == 0) {
+      throw new DlAbortEx(EX_SSL_INIT_FAILURE);
+    }
+     // TODO handling return value == 0 case required
+    if(SSL_connect(ssl) <= 0) {
+      throw new DlAbortEx(EX_SSL_INIT_FAILURE);
+    }
+    secure = true;
+  }
+}
+#endif // HAVE_LIBSSL

src/SocketCore.h

@@ -23,6 +23,12 @@
 #define _D_SOCKET_CORE_H_
 
 #include <string>
+#include "common.h"
+
+#ifdef HAVE_LIBSSL
+// for SSL
+# include <openssl/ssl.h>
+#endif // HAVE_LIBSSL
 
 using namespace std;
 
@@ -33,6 +39,12 @@ private:
   int sockfd;
   // reference counter for this object.
   int use;
+  bool secure;
+#ifdef HAVE_LIBSSL
+  // for SSL
+  SSL_CTX* sslCtx;
+  SSL* ssl;
+#endif // HAVE_LIBSSL
 public:
   SocketCore();
   ~SocketCore();
@@ -71,6 +83,13 @@ public:
   // Reads up to len bytes from this socket, but bytes are not removed from
   // this socket.
   void peekData(char* data, int& len, int timeout = 5);
+  
+#ifdef HAVE_LIBSSL
+  /**
+   * Makes this socket SSL endpoint
+   */
+  void initiateSecureConnection();
+#endif // HAVE_LIB_SSL
 };
 
 #endif // _D_SOCKET_CORE_H_

src/main.cc

@@ -37,6 +37,12 @@ extern char* optarg;
 extern int optind, opterr, optopt;
 #include <getopt.h>
 
+#ifdef HAVE_LIBSSL
+// for SSL
+# include <openssl/err.h>
+# include <openssl/ssl.h>
+#endif // HAVE_LIBSSL
+
 using namespace std;
 
 void clearRequest(Request* req) {
@@ -243,6 +249,11 @@ int main(int argc, char* argv[]) {
       exit(1);
     }
   }
+#ifdef HAVE_LIBSSL
+  // for SSL initialization
+  SSL_load_error_strings();
+  SSL_library_init();
+#endif // HAVE_LIBSSL
   SimpleLogger* logger;
   if(stdoutLog) {
     logger = new SimpleLogger(stdout);

src/message.h

@@ -51,5 +51,5 @@
 #define EX_BAD_STATUS "The response status is not successful. status = %d"
 #define EX_TOO_LARGE_FILE "Too large file size. size = %d"
 #define EX_TRANSFER_ENCODING_NOT_SUPPORTED "Transfer encoding %s is not supported."
-
+#define EX_SSL_INIT_FAILURE "SSL initialization failed."
 #endif // _D_MESSAGE_H_