NetworkProtocols.cpp

Go to the documentation of this file.

#include "NetworkProtocols.h"
#include "PsyTime.h"
#include "UnitTestFramework.h"
 
namespace   cmlabs{

// Data Types
 
//HTTPRequest::HTTPRequest(uint64 startRecTime) {
//  if (startRecTime)
//      time = startRecTime;
//  else
//      time = GetTimeNow();
//  endReceiveTime = 0;
//  this->source = 0;
//  type = 0;
//  headerLength = 0;
//  contentLength = 0;
//  ifModifiedSince = 0;
//  keepAlive = true;
//  data = NULL;
//}
 
HTTPRequest* HTTPRequest::CreateWebsocketRequest(const char* uri, const char* host, const char* protocolName, const char* origin) {
    HTTPRequest* req = new HTTPRequest();
    if (req->createWebsocketRequest(uri, host, protocolName, origin))
        return req;
    else {
        delete req;
        return NULL;
    }
}
 
HTTPRequest::HTTPRequest(uint64 source, uint64 startRecTime) {
    if (startRecTime)
        time = startRecTime;
    else
        time = GetTimeNow();
    endReceiveTime = 0;
    this->source = source;
    type = 0;
    headerLength = 0;
    contentLength = 0;
    ifModifiedSince = 0;
    keepAlive = true;
    data = NULL;
}
 
HTTPRequest::HTTPRequest(HTTPRequest* req) {
    time = req->time;
    endReceiveTime = req->endReceiveTime;
    source = req->source;
    type = req->type;
    headerLength = req->headerLength;
    contentLength = req->contentLength;
    ifModifiedSince = req->ifModifiedSince;
    keepAlive = req->keepAlive;
    data = new char[headerLength+contentLength+1];
    memcpy(data, req->data, headerLength+contentLength+1);
    entries = req->entries;
    params = req->params;
}
 
 
HTTPRequest::~HTTPRequest() {
    if (data != NULL)
        delete(data);
    data = NULL;
    std::map<std::string, HTTPPostEntry*>::iterator i = postEntries.begin(), e = postEntries.end();
    while (i != e) {
        delete(i->second);
        i++;
    }
    postEntries.clear();
}
 
// Reading it in from a socket
bool HTTPRequest::processHeader(const char* buffer, uint32 size, bool &isInvalid) {
    isInvalid = false;
    if (!buffer || (size < 5))
        return false;
 
    if (utils::stristr(buffer, "GET ") == buffer)
        type = HTTP_GET;
    else if (utils::stristr(buffer, "PUT ") == buffer)
        type = HTTP_PUT;
    else if (utils::stristr(buffer, "POST ") == buffer)
        type = HTTP_POST;
    else if (utils::stristr(buffer, "HEAD ") == buffer)
        type = HTTP_HEAD;
    else if (utils::stristr(buffer, "DELETE ") == buffer)
        type = HTTP_DELETE;
    else if (utils::stristr(buffer, "OPTIONS ") == buffer)
        type = HTTP_OPTIONS;
    else {
        isInvalid = true;
        return false;
    }
 
    // Find end of header
    uint32 n;
    for (n = 3; n < size; n++) {
        if (buffer[n] == 10) {
            if ( (size-n >= 2) && (buffer[n+1] == 10) ) {
                // Just using LF, no CR
                headerLength = n+2;
                break;
            }
            else if ( (size-n >= 3) && (buffer[n+1] == 13) && (buffer[n+2] == 10) && (buffer[n-1] == 13) ) {
                // Using CRLF
                headerLength = n+3;
                break;
            }
        }
    }
 
    if (headerLength == 0)
        return false;
 
    uint32 tempContentLength = 1024;
    data = new char[headerLength+tempContentLength+1];
    memcpy(data, buffer, headerLength);
    data[headerLength] = 0; // temp sting end
 
    // Now process header entries
    std::string key, val, uri;
    const char* s = data, *t;
    n = 0;
    uint32 crSize = 0;
    uint32 left = headerLength;
    size_t i, j;
    while (n < headerLength) {
        if (!utils::GetNextLineEnd(s, left, n, crSize))
            break;
        // Process line
        if ((t = strchr(s, ' ')) && (t - s < (int32)n)) {
            t -= 1;
            if (*t != ':') {
                t++;
                key.assign(s, t - s);
            }
            else {
                key.assign(s, t - s);
                t++;
            }
            while (*t == 32) t++;
            if (*t > 32)
                val.assign(t, n + s - t);
            else
                val = "";
        }
        else {
            key.assign(s, n);
            val = "";
        }
        // Record entry
        if (s == data) {
            entries["http_fullrequest"] = val;
            if ((i = val.find(' ')) != std::string::npos) {
                uri = val.substr(0, i);
                entries["http_uri"] = uri;
                //printf("*** URI: %s\n", uri.c_str());
                entries["http_protocol"] = val.substr(i+1);
                if ((j = uri.find('?')) != std::string::npos) {
                    entries["http_request"] = uri.substr(1, j-1); // without the first /
                    parseURIParameters(uri.substr(j+1).c_str());
                }
                else
                    entries["http_request"] = uri.substr(1, i-1); // without the first /
            }
            else {
                uri = val;
                entries["http_uri"] = uri;
                entries["http_protocol"] = "";
                if ((j = uri.find('?')) != std::string::npos) {
                    entries["http_request"] = uri.substr(1, j-1); // without the first /
                    parseURIParameters(uri.substr(j+1).c_str());
                }
                else
                    entries["http_request"] = uri.substr(1); // without the first /
            }
        }
        else if (key.length() > 0) {
            if (entries.find(key) != entries.end())
                entries[key] = entries[key] + ";" + val;
            else
                entries[key] = val;
            // Check entry
            if (stricmp(key.c_str(), "content-length") == 0)
                contentLength = atoi(val.c_str());
            else if (stricmp(key.c_str(), "if-modified-since") == 0)
                ifModifiedSince = GetTimeFromString(val.c_str());
            else if (stricmp(key.c_str(), "connection") == 0)
                keepAlive = (stricmp(val.c_str(), "close") != 0);
            else if (stricmp(key.c_str(), "content-type") == 0) {
                std::multimap<std::string,std::string> conTypes = utils::TextMultiMapSplit(val.c_str(), ";", "=");
                std::multimap<std::string,std::string>::iterator ci = conTypes.find("boundary");
                if (ci != conTypes.end())
                    postBoundary = utils::StringFormat("--%s", ci->second.c_str());
            }
        }
        // Next line
        s += n + crSize;
        left -= n + crSize;
    }
 
    if (contentLength > tempContentLength) {
        delete(data);
        data = new char[headerLength+contentLength+1];
        memcpy(data, buffer, headerLength);
        data[headerLength] = 0; // temp string end
        data[headerLength+contentLength] = 0;
    }
 
    if (!contentLength)
        endReceiveTime = GetTimeNow();
 
    return true;
}
 
bool HTTPRequest::processContent(const char* buffer, uint32 size) {
    if (!buffer || (size != contentLength))
        return false;
    memcpy(data+headerLength, buffer, size);
    data[headerLength+contentLength] = 0;
    parseContentParameters(data+headerLength, size);
    endReceiveTime = GetTimeNow();
    return true;
}
 
const char* HTTPRequest::getContent(uint32& size) {
    size = contentLength;
    return data+headerLength;
}
 
const char* HTTPRequest::getRawContent(uint32& size) {
    size = contentLength + headerLength;
    return data;
}
 
const char* HTTPRequest::getHeaderEntry(const char* entry) {
    std::map<std::string, std::string>::iterator it = entries.find(entry);
    if (it != entries.end())
        return it->second.c_str();
    else
        return NULL;
}
 
const char* HTTPRequest::getRequest() {
    return getHeaderEntry("http_request");
}
 
const char* HTTPRequest::getURI() {
    return getHeaderEntry("http_uri");
}
 
const char* HTTPRequest::getProtocol() {
    return getHeaderEntry("http_protocol");
}
 
const char* HTTPRequest::getBasicAuthorization() {
    const char* auth = getHeaderEntry("Authorization");
    if (!auth || !utils::TextStartsWith(auth, "Basic ", false))
        return NULL;
    return auth + 6; // after 'Basic '
}
 
std::string HTTPRequest::decodeBasicAuthorization() {
    const char* auth = getBasicAuthorization();
    if (!auth) return "";
    std::string authString = base64_decode(auth);
    if (!authString.length() || (authString.find(":") == std::string::npos))
        return "";
    return authString;
}
 
std::string HTTPRequest::getBasicAuthorizationUser() {
    std::string authString = decodeBasicAuthorization();
    if (!authString.length())
        return "";
    return utils::TextListSplit(authString.c_str(), ":").at(0);
}
 
std::string HTTPRequest::getBasicAuthorizationPassword() {
    std::string authString = decodeBasicAuthorization();
    if (!authString.length())
        return "";
    return utils::TextListSplit(authString.c_str(), ":").at(1);
}
 
bool HTTPRequest::setBasicAuthorization(const char* authB64) {
    if (!authB64 || !strlen(authB64)) {
        if (entries.find("Authorization") != entries.end())
            entries.erase("Authorization");
    }
    else {
        entries["Authorization"] = authB64;
    }
    return true;
}
 
bool HTTPRequest::setBasicAuthorization(const char* user, const char* password) {
    if (!user || !strlen(user) || !password || !strlen(password))
        return false;
    std::string rawAuth = utils::StringFormat("%s:%s", user, password);
    std::string auth = base64_encode(rawAuth);
    entries["Authorization"] = auth;
    return true;
}
 
bool HTTPRequest::parseContentParameters(const char* content, uint32 size) {
//  "------WebKitFormBoundarynP2yLpjkwmT3R8j5\r\nContent-Disposition: form-data; name="Username"\r\n\r\nundefined\r\n"
//  "------WebKitFormBoundarynP2yLpjkwmT3R8j5\r\nContent-Disposition: form-data; name="Password"\r\n\r\nundefined\r\n"
//  "------WebKitFormBoundarynP2yLpjkwmT3R8j5\r\nContent-Disposition: form-data; name="Authentication"\r\n\r\nundefined\r\n"
//  "------WebKitFormBoundarynP2yLpjkwmT3R8j5--\r\n"
    if (!content)
        return false;
 
    // check main content-type
    std::string contentType = entries["Content-Type"];
 
    if (contentType.find("application/x-www-form-urlencoded") != std::string::npos) {
        // post just contains one content chunk
        parseContentChunk(content, size);
    }
    else if (contentType.find("multipart/form-data") != std::string::npos) {
        // Check if using boundaries
        if (!postBoundary.length() || (size < (2 * postBoundary.length())) )
            return false;
        //printf("[%u] %s\n", size, content);
        uint32 boundarySize = (uint32)postBoundary.length();
 
        // Check content
        const char* boundary = postBoundary.c_str();
        if (memcmp(content, boundary, boundarySize) != 0)
            return false;
 
        const char* lastStart = content + boundarySize + 2;
        const char* c = lastStart;
 
        for (uint32 n=boundarySize; n<size; n++) {
            //if ((*c == '-') && (memcmp(c, boundary, boundarySize) == 0)) {
            if (memcmp(c, boundary, boundarySize) == 0) {
                parseContentChunk(lastStart, (uint32)(c - lastStart));
                lastStart = c += boundarySize + 2;
                n += boundarySize + 2;
            }
            else
                c++;
        }
    }
    else {
        // Post content is something else
        HTTPPostEntry* postEntry = new HTTPPostEntry();
        postEntry->name = "POST";
        postEntry->type = contentType;
        postEntry->setContent(content, size);
        postEntries[postEntry->name] = postEntry;
    }
 
 
    return true;
}
 
bool HTTPRequest::parseContentChunk(const char* chunk, uint32 size) {
//  "Content-Disposition: form-data; name="Username"\r\n\r\nundefined\r\n"
 
    const char* valContent = strstr(chunk, "\r\n\r\n");
    if (!valContent)
        return false;
    uint32 headerSize = (uint32)(valContent-chunk);
    valContent += 4;
    uint32 valContentSize = size - headerSize - 6;
 
    HTTPPostEntry* postEntry = new HTTPPostEntry();
    char* headerText = new char[headerSize+1];
    memcpy(headerText, chunk, headerSize);
    headerText[headerSize] = 0;
 
    //printf("*** %s\n", headerText);
 
    std::multimap<std::string, std::string> entryMap;
    std::multimap<std::string, std::string>::iterator ei;
 
    std::vector<std::string> headerEntries = utils::TextListSplit(headerText, "\r\n", true, true);
    std::vector<std::string>::iterator i = headerEntries.begin(), e = headerEntries.end();
    while (i != e) {
        if (utils::TextStartsWith((*i).c_str(), "Content-Disposition:", false)) {
            entryMap = utils::TextMultiMapSplit((*i).c_str(), ";", "=");
            if ( (ei = entryMap.find("name")) != entryMap.end())
                postEntry->name = utils::TextTrimQuotes(ei->second.c_str());
            if ( (ei = entryMap.find("filename")) != entryMap.end())
                postEntry->filename = utils::TextTrimQuotes(ei->second.c_str());
        }
        else if (utils::TextStartsWith((*i).c_str(), "Content-Type:", false)) {
            postEntry->type = (*i).substr(14);
        }
        i++;
    }
    delete [] headerText;
    if (postEntry->name.length())
        postEntry->setContent(valContent, valContentSize);
 
    if (postEntry->isValid()) {
        postEntries[postEntry->name] = postEntry;
        return true;
    }
    else {
        delete(postEntry);
        return false;
    }
 
 
    //CGIRequestEntry reqEntry;
 
    //int pos;
    //QStringList entries;
    //for (int n=0; n<headerList.size(); n++) {
    //  if ((entry = headerList.at(n).trimmed()).length()) {
    //      if (entry.startsWith("Content-Disposition:", Qt::CaseInsensitive)) {
    //          entries = entry.split(";");
    //          for (int m=0; m<entries.size(); m++) {
    //              if ( (pos = entries.at(m).indexOf(" name=")) >= 0) {
    //                  reqEntry.name = entries.at(m).mid(pos+6).replace("\"","");
    //              }
    //              else if ( (pos = entries.at(m).indexOf(" filename=")) >= 0) {
    //                  reqEntry.filename = entries.at(m).mid(pos+10).replace("\"","");
    //              }
    //          }
    //      }
    //      else if (entry.startsWith("Content-Type:", Qt::CaseInsensitive)) {
    //          reqEntry.type = entry.split(":").at(1).trimmed().replace("\"","");
    //      }
    //  }
    //}
 
    //if (!reqEntry.name.length())
    //  return false;
 
    //reqEntry.data = dataChunk.mid(doubleBreakPos+4);
    //requestEntries[reqEntry.name] = reqEntry;
    //return true;
}
 
bool HTTPRequest::parseURIParameters(const char* text) {
    if (!text)
        return false;
 
    std::multimap<std::string, std::string> paramMap = utils::TextMultiMapSplit(text, "&", "=");
    std::multimap<std::string, std::string>::iterator i = paramMap.begin(), e = paramMap.end();
    while (i != e) {
        params[html::DecodeHTML(i->first)] = html::DecodeHTML(i->second);
        i++;
    }
    return true;
 
//  uint32 len = (uint32)strlen(text);
//  const char* s = text;
//  const char* e = strchr(s, '=');
//  const char* p = strchr(s, '&');
 
//  if (!e && !p) {
//      key.assign(s);
//      params[html::DecodeHTML(key)] = "";
//      return true;
//  }
 
//  std::string key, val;
 
//  while ( e ) {
//      key.assign(s, e-s);
//      if (p) {
//          val.assign(e+1, p-e-1);
//          params[html::DecodeHTML(key)] = html::DecodeHTML(val);
//          s = p+1;
//          e = strchr(s, '=');
//          p = strchr(s, '&');
//      }
//      else {
//          val.assign(e+1);
//          params[html::DecodeHTML(key)] = html::DecodeHTML(val);
//          break;
//      }
//  }
//  return true;
}
 
DataMessage* HTTPRequest::convertToMessage() {
    DataMessage* msg = new DataMessage();
    std::map<std::string, std::string>::iterator i, e;
 
    msg->setInt("HTTP_OPERATION", type);
    msg->setString("REQUEST", this->getRequest());
    msg->setString("URI", this->getURI());
    msg->setTime("SOURCE", this->source);
 
    for (i=entries.begin(), e=entries.end(); i!=e; i++)
        msg->setString(i->first.c_str(), i->second.c_str());
 
    for (i=params.begin(), e=params.end(); i!=e; i++)
        msg->setString(i->first.c_str(), i->second.c_str());
 
    std::map<std::string, HTTPPostEntry*>::iterator pi = postEntries.begin(), pe = postEntries.end();
    while (pi != pe) {
        if (pi->second->type.length()) {
            msg->setString((pi->first+"_CONTENT_TYPE_").c_str(), pi->second->type.c_str());
            if (utils::stristr(pi->second->type.c_str(), "json"))
                msg->setString(pi->first.c_str(), pi->second->content);
            else if (utils::stristr(pi->second->type.c_str(), "xml"))
                msg->setString(pi->first.c_str(), pi->second->content);
            else if (utils::stristr(pi->second->type.c_str(), "text"))
                msg->setString(pi->first.c_str(), pi->second->content);
            else
                msg->setData(pi->first.c_str(), pi->second->content, pi->second->contentSize);
        }
        else
            msg->setString(pi->first.c_str(), pi->second->content);
        pi++;
    }
    msg->setCreatedTime(time);
    msg->setRecvTime(endReceiveTime);
 
    return msg;
}
 
bool HTTPRequest::isWebsocketUpgrade() {
    const char* connection = this->getHeaderEntry("Connection");
    if (!connection || stricmp(connection, "Upgrade"))
        return false;
 
    const char* upgrade = this->getHeaderEntry("Upgrade");
    if (!upgrade || stricmp(upgrade, "websocket"))
        return false;
 
    return true;
}
 
 
 
const char* HTTPRequest::getParameter(const char* entry) {
    std::map<std::string, std::string>::iterator it = params.find(entry);
    if (it != params.end())
        return it->second.c_str();
    else
        return NULL;
}
 
const char* HTTPRequest::getPostData(const char* entry, uint32& size, const char** type) {
    std::map<std::string, HTTPPostEntry*>::iterator i = postEntries.find(entry);
    if (i == postEntries.end())
        return NULL;
    size = i->second->contentSize;
    *type = i->second->type.c_str();
    return i->second->content;
}
 
const char* HTTPRequest::getPostData(const char* entry, uint32& size) {
    std::map<std::string, HTTPPostEntry*>::iterator i = postEntries.find(entry);
    if (i == postEntries.end())
        return NULL;
    size = i->second->contentSize;
    return i->second->content;
}
 
const char* HTTPRequest::getPostDataType(const char* entry) {
    std::map<std::string, HTTPPostEntry*>::iterator i = postEntries.find(entry);
    if (i == postEntries.end())
        return NULL;
    return i->second->type.c_str();
}
 
 
bool HTTPRequest::createRequest(
    uint8 type, const char* host, const char* uri, const char* content,
    uint32 contentSize, bool keepAlive, uint64 ifModifiedSince) {
 
    char timeString[1024];
    if (!ifModifiedSince) {
        if (!GetHTTPTime(TIME_YEAR_1970, timeString, 1024))
            timeString[0] = 0;
    }
    else {
        if (!GetHTTPTime(ifModifiedSince, timeString, 1024))
            timeString[0] = 0;
    }
 
    if (data)
        delete(data);
    data = new char[4096 + contentSize];
    if (!content) {
        snprintf(data, 4096+contentSize,
"%s %s HTTP/1.1\r\n\
Host: %s\r\n\
Accept: */*\r\n\
User-Agent: CMLabsHTTP/1.0\r\n\
if-modified-since: %s\r\n\r\n",
            HTTP_Type[type], uri, host, timeString);
    }
    else {
        snprintf(data, 4096+contentSize,
"%s %s HTTP/1.1\r\n\
Host: %s\r\n\
Accept: */*\r\n\
User-Agent: CMLabsHTTP/1.0\r\n\
Content-Type: application/x-www-form-urlencoded\r\n\
Content-Length: %u\r\n\r\n",
            HTTP_Type[type], uri, host, (uint32) (contentSize ? contentSize : strlen(content)));
    }
 
    headerLength = (uint32)strlen(data);
    memcpy(data+headerLength, content, contentSize);
    contentLength = contentSize;
    data[headerLength + contentLength] = 0;
 
    return true;
}
 
bool HTTPRequest::createRequest(uint8 type, const char* host, const char* uri, std::map<std::string, std::string>& headerEntries, bool keepAlive, uint64 ifModifiedSince) {
 
    // First create the header
 
    char timeString[1024];
    if (!ifModifiedSince) {
        if (!GetHTTPTime(TIME_YEAR_1970, timeString, 1024))
            timeString[0] = 0;
    }
    else {
        if (!GetHTTPTime(ifModifiedSince, timeString, 1024))
            timeString[0] = 0;
    }
 
    std::string headerContent = utils::StringFormat(
        "%s %s HTTP/1.1\r\n\
Host: %s\r\n\
Accept: */*\r\n\
User-Agent: CMLabsHTTP/1.0\r\n\
if-modified-since: %s\r\n",
HTTP_Type[type], uri, host, timeString);
 
    std::map<std::string, std::string>::iterator hI = headerEntries.begin(), hE = headerEntries.end();
    while (hI != hE) {
        if ((stricmp(hI->first.c_str(), "content-type") != 0) &&
            (stricmp(hI->first.c_str(), "content-length") != 0)) {
            headerContent += utils::StringFormat("%s: %s\r\n", hI->first.c_str(), hI->second.c_str());
        }
        hI++;
    }
 
    // Calculate content size
    headerLength = (uint32)headerContent.length();
 
    if (data)
        delete(data);
    data = new char[headerLength + 1];
 
    memcpy(data, headerContent.c_str(), headerLength);
    data[headerLength] = 0;
    return true;
}
 
 
bool HTTPRequest::createRequest(uint8 type, const char* host, const char* uri, std::map<std::string, std::string>& headerEntries, HTTPPostEntry* bodyEntry, bool keepAlive, uint64 ifModifiedSince) {
    if (!bodyEntry)
        return createRequest(type, host, uri, headerEntries, keepAlive, ifModifiedSince);
    else
        return createRequest(type, host, uri, headerEntries, bodyEntry->content, bodyEntry->type.c_str(), bodyEntry->contentSize, keepAlive, ifModifiedSince);
}
 
bool HTTPRequest::createRequest(uint8 type, const char* host, const char* uri, std::map<std::string, std::string>& headerEntries, const char* content, const char* contentType, uint32 contentSize, bool keepAlive, uint64 ifModifiedSince) {
 
    if (!contentSize)
        return createRequest(type, host, uri, headerEntries, keepAlive, ifModifiedSince);
 
    if (!content || !contentType)
        return false;
 
    // First create the header
    char timeString[1024];
    if (!ifModifiedSince) {
        if (!GetHTTPTime(TIME_YEAR_1970, timeString, 1024))
            timeString[0] = 0;
    }
    else {
        if (!GetHTTPTime(ifModifiedSince, timeString, 1024))
            timeString[0] = 0;
    }
 
    std::string headerContent = utils::StringFormat(
        "%s %s HTTP/1.1\r\n\
Host: %s\r\n\
Accept: */*\r\n\
User-Agent: CMLabsHTTP/1.0\r\n\
Content-Type: %s\r\n\
if-modified-since: %s\r\n",
HTTP_Type[type], uri, host, contentType, timeString);
 
    std::map<std::string, std::string>::iterator hI = headerEntries.begin(), hE = headerEntries.end();
    while (hI != hE) {
        if ((stricmp(hI->first.c_str(), "content-type") != 0) &&
            (stricmp(hI->first.c_str(), "content-length") != 0)) {
            headerContent += utils::StringFormat("%s: %s\r\n", hI->first.c_str(), hI->second.c_str());
        }
        hI++;
    }
 
    // Calculate content size
    contentLength = contentSize;
    headerContent += utils::StringFormat("Content-Length: %u\r\n\r\n", contentLength);
    headerLength = (uint32)headerContent.length();
 
    if (data)
        delete(data);
    data = new char[headerLength + contentLength + 1];
 
    memcpy(data, headerContent.c_str(), headerLength);
    char* dst = data + headerLength;
 
    memcpy(dst, content, contentLength);
    data[headerLength + contentLength] = 0;
    return true;
}
 
 
bool HTTPRequest::createMultipartRequest(
    uint8 type, const char* host, const char* uri, std::map<std::string, std::string>& headerEntries,
    std::map<std::string, HTTPPostEntry*>& bodyEntries, bool keepAlive, uint64 ifModifiedSince) {
 
    // First create the header
 
    std::string boundaryString = utils::StringFormat("----cmboundary%lld", utils::RandomInt(0, 10000000));
    uint32 boundaryLength = (uint32)boundaryString.length();
 
    char timeString[1024];
    if (!ifModifiedSince) {
        if (!GetHTTPTime(TIME_YEAR_1970, timeString, 1024))
            timeString[0] = 0;
    }
    else {
        if (!GetHTTPTime(ifModifiedSince, timeString, 1024))
            timeString[0] = 0;
    }
 
    std::string headerContent = utils::StringFormat(
        "%s %s HTTP/1.1\r\n\
Host: %s\r\n\
Accept: */*\r\n\
User-Agent: CMLabsHTTP/1.0\r\n\
Content-Type: multipart/form-data; boundary=%s\r\n\
if-modified-since: %s\r\n",
            HTTP_Type[type], uri, host, boundaryString.c_str(), timeString);
 
    std::map<std::string, std::string>::iterator hI = headerEntries.begin(), hE = headerEntries.end();
    while (hI != hE) {
        if ((stricmp(hI->first.c_str(), "content-type") != 0) &&
                (stricmp(hI->first.c_str(), "content-length") != 0)) {
            headerContent += utils::StringFormat("%s: %s\r\n", hI->first.c_str(), hI->second.c_str());
        }
        hI++;
    }
 
    // Remember to add this to the header at the end... Content-Length: %u\r\n\r\n",
 
    // Calculate content size
    contentLength = 2 + boundaryLength + 2; // Add the first --boundary\r\n
 
    uint32 filenameStatementLength = 0;
 
    std::map<std::string, HTTPPostEntry*>::iterator bI = bodyEntries.begin(), bE = bodyEntries.end();
    while (bI != bE) {
        if (html::IsMimeBinary(bI->second->type.c_str()))
            filenameStatementLength = 19;
        else
            filenameStatementLength = 0;
        // Content-Type: image/jpeg\r\n
        // Content-Disposition: form-data; name="image"; filename="nofile"\r\n\r\n
        // <content>\r\n
        // --boundary
        contentLength += 14 + (uint32)bI->second->type.length() + 2
            + 38 + (uint32)bI->second->name.length() + 5 + filenameStatementLength
            + bI->second->contentSize + 2 + 2 + boundaryLength;
        if (bI != bE) {
            contentLength += 2;
        }
        bI++;
    }
    contentLength += 4; // for the last --\r\n
 
    headerContent += utils::StringFormat("Content-Length: %u\r\n\r\n", contentLength);
    headerLength = (uint32)headerContent.length();
 
    if (data)
        delete(data);
    data = new char[headerLength + contentLength + 1];
 
    memcpy(data, headerContent.c_str(), headerLength);
    char* dst = data + headerLength;
 
    //sprintf(dst, "--%s\r\n", boundaryString.c_str());
    snprintf(dst, 2 + boundaryLength + 2, "--%s\r\n", boundaryString.c_str());
    dst += 2 + boundaryLength + 2;
 
 
    bI = bodyEntries.begin();
    while (bI != bE) {
        // Content-Type: image/jpeg\r\n
        // Content-Disposition: form-data; name="image"; filename="nofile"\r\n\r\n
        // <content>\r\n
        // --boundary
        if (html::IsMimeBinary(bI->second->type.c_str())) {
            filenameStatementLength = 19;
            // sprintf(dst, "Content-Type: %s\r\nContent-Disposition: form-data; name=\"%s\"; filename=\"nofile\"\r\n\r\n",
            snprintf(dst, 14 + bI->second->type.length() + 2 + 38 + bI->second->name.length() + 5 + 19, "Content-Type: %s\r\nContent-Disposition: form-data; name=\"%s\"; filename=\"nofile\"\r\n\r\n",
                bI->second->type.c_str(),
                bI->second->name.c_str());
        }
        else {
            // sprintf(dst, "Content-Type: %s\r\nContent-Disposition: form-data; name=\"%s\"\r\n\r\n",
            snprintf(dst, 14 + bI->second->type.length() + 2 + 38 + bI->second->name.length() + 5, "Content-Type: %s\r\nContent-Disposition: form-data; name=\"%s\"\r\n\r\n",
                bI->second->type.c_str(),
                bI->second->name.c_str());
            filenameStatementLength = 0;
        }
 
        dst += 14 + bI->second->type.length() + 2
            + 38 + bI->second->name.length() + 5 + filenameStatementLength;
        memcpy(dst, bI->second->content, bI->second->contentSize);
        dst += bI->second->contentSize;
        // sprintf(dst, "\r\n--%s", boundaryString.c_str());
        snprintf(dst, 2 + 2 + boundaryLength, "\r\n--%s", boundaryString.c_str());
        dst += 2 + 2 + boundaryLength;
        bI++;
        if (bI != bE) {
            // sprintf(dst, "\r\n");
            snprintf(dst, 4, "\r\n"); // put 4 in instead of 2 to avoid compiler warning
            dst += 2;
        }
    }
    // sprintf(dst, "--\r\n");
    snprintf(dst, 6, "--\r\n");  // put 6 in instead of 4 to avoid compiler warning
    dst += 4;
    data[headerLength + contentLength] = 0;
    //uint32 check = (uint32)(dst - data);
    //uint32 check2 = headerLength + contentLength;
    //printf("Memory size: %u   -   data size: %u\n", check2, check);
    return true;
}
 
bool HTTPRequest::createWebsocketRequest(const char* uri, const char* host, const char* protocolName, const char* origin) {
 
    //GET /chat HTTP/1.1
    //Host: server.example.com
    //Upgrade: websocket
    //Connection: Upgrade
    //Sec-WebSocket-Key: x3JJHMbDL1EzLkh9GBhXDw==
    //Sec-WebSocket-Protocol: chat, superchat
    //Sec-WebSocket-Version: 13
    //Origin: http://example.com
 
    if (!host)
        return false;
 
    std::string uriString;
    if (uri && strlen(uri))
        uriString = uri;
    if (!utils::TextStartsWith(uriString.c_str(), "/", false))
        uriString = utils::StringFormat("/%s", uriString.c_str());
 
    std::string originString;
    if (origin && strlen(origin))
        originString = origin;
    else
        originString = utils::StringFormat("http://%s/", host);
 
    std::string protocolNameString;
    if (protocolName && strlen(protocolName))
        protocolNameString = protocolName;
    else
        protocolNameString = "Default";
 
    char* randomKey = new char[17];
    for (uint32 n = 0; n < 16; n++)
        randomKey[n] = (uint8)utils::RandomInt(33, 255);
    randomKey[16] = 0;
    std::string key = base64_encode(randomKey);
    //std::string key = "x3JJHMbDL1EzLkh9GBhXDw==";
 
    if (data)
        delete(data);
    data = new char[4096];
    snprintf(data, 4096,
"GET %s HTTP/1.1\r\n\
Host: %s\r\n\
Upgrade: websocket\r\n\
Connection: Upgrade\r\n\
Sec-WebSocket-Key: %s\r\n\
Sec-WebSocket-Protocol: %s\r\n\
Sec-WebSocket-Version: 13\r\n\
Origin: %s\r\n\r\n",
        uriString.c_str(), host, key.c_str(), protocolNameString.c_str(), originString.c_str());
 
    headerLength = (uint32)strlen(data);
    contentLength = 0;
    data[headerLength] = 0;
    return true;
}
 
 
uint32 HTTPRequest::getSize() {
    if (data)
        return headerLength + contentLength;
    else
        return 0;
}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
WebsocketData::WebsocketData(uint64 source, uint64 startRecTime) {
    this->source = source;
    headerLength = 0;
    payloadSize = contentSize = 0;
    opcode = 0;
    maskingKey = 0;
    packages = 0;
    data = NULL;
    rawData = NULL;
    dataType = NONE;
    status = IDLE;
    isFinal = false;
}
 
WebsocketData::WebsocketData(WebsocketData* wsData) {
    headerLength = wsData->headerLength;
    payloadSize = wsData->payloadSize;
    contentSize = wsData->contentSize;
    opcode = wsData->opcode;
    maskingKey = wsData->maskingKey;
    packages = wsData->packages;
    if (wsData->data) {
        data = new char[(uint32)payloadSize + 1];
        memcpy(data, wsData->data, (uint32)payloadSize);
        data[payloadSize] = 0;
    }
    dataType = wsData->dataType;
    if (wsData->rawData) {
        rawData = new char[(uint32)payloadSize + headerLength + 1];
        memcpy(rawData, wsData->rawData, (uint32)payloadSize + headerLength);
        rawData[payloadSize + headerLength] = 0;
    }
    status = wsData->status;
    isFinal = wsData->isFinal;
}
 
WebsocketData::~WebsocketData() {
    if (data != NULL)
        delete(data);
    data = NULL;
    if (rawData != NULL)
        delete(rawData);
    rawData = NULL;
    status = IDLE;
}
 
WebsocketData* WebsocketData::CreateTerminationConfirmation() {
    WebsocketData* wsData = new WebsocketData();
    wsData->setData(wsData->CLOSE, false);
    return wsData;
}
 
WebsocketData* WebsocketData::CreatePing() {
    WebsocketData* wsData = new WebsocketData();
    wsData->setData(wsData->PING, false);
    return wsData;
}
 
WebsocketData* WebsocketData::CreatePong() {
    WebsocketData* wsData = new WebsocketData();
    wsData->setData(wsData->PONG, false);
    return wsData;
}
 
bool WebsocketData::isTerminationRequest() {
    return (dataType == CLOSE);
}
 
bool WebsocketData::isTerminationConfirmation() {
    return (dataType == CLOSE);
}
 
bool WebsocketData::isPing() {
    return (dataType == PING);
}
 
bool WebsocketData::isPong() {
    return (dataType == PONG);
}
 
bool WebsocketData::setData(DataType dataType, bool maskData, const char* data, uint64 size) {
    // Builds the complete RFC 6455 frame into rawData:
    //   byte 0: FIN | RSV1-3 | opcode        (we always send FIN = single frame)
    //   byte 1: MASK | 7-bit payload length  (126 => 16-bit ext, 127 => 64-bit ext)
    //   [2 or 8 bytes extended length, network byte order]
    //   [4-byte masking key when maskData]   (clients MUST mask per the RFC)
    //   payload (XOR-masked with the key when maskData)
    if ((dataType == NONE))
        return false;
 
    this->dataType = dataType;
    if (maskData && !maskingKey)
        maskingKey = (uint32)utils::RandomInt(0, MAXVALUINT32);
 
    headerLength = basic_header_length; // +sizeof(uint32);
    if (maskData)
        headerLength += sizeof(uint32);
    uint32 mkOffset = basic_header_length;
    uint32 basic_value = 0;
 
    if (size <= payload_size_basic) {
        basic_value = (uint8)size;
    }
    else if (size <= payload_size_extended) {
        mkOffset += 2;
        headerLength += 2;
        basic_value = payload_size_code_16bit;
    }
    else {
        headerLength += 8;
        mkOffset += 8;
        basic_value = payload_size_code_64bit;
    }
 
    if (rawData)
        delete rawData;
 
    uint8 opcode = (uint8)dataType;
 
    rawData = new char[headerLength + (uint32)size + 1];
    memset(rawData, 0, headerLength);
 
    uint8* b0 = (uint8*)rawData;
    uint8* b1 = (uint8*)(rawData + 1);
 
    *b0 |= BHB0_FIN;
    *b0 |= (opcode & BHB0_OPCODE);
    if (maskData)
        *b1 |= BHB1_MASK;
 
    *b1 |= basic_value;
 
    if (maskData)
        *(uint32*)(rawData + mkOffset) = maskingKey;
 
    contentSize = payloadSize = size;
 
    if (!size)
        return true;
 
    // Write the extended length field. NOTE: the branch ordering here means the
    // 64-bit ("jumbo", > 65535 bytes) case is unreachable — any size above 125
    // takes the first branch and is truncated to 16 bits by ntohs((uint16)size).
    // Payloads up to 64KB (the sizes actually used by the SDK) are unaffected;
    // larger single frames would be framed incorrectly. (ntohs is used for
    // host->network conversion; it is its own inverse, so this is equivalent
    // to htons.)
    if (size > payload_size_basic) {
        uint16* h2 = (uint16*)(rawData + 2);
        *h2 = ntohs((uint16)size);
    }
    else if (size > payload_size_extended) {
        uint64* h3 = (uint64*)(rawData + 2);
        *h3 = utils::ntoh64(&size);
    }
 
    if (maskData) {
        // RFC 6455 masking: payload byte i is XORed with key byte (i mod 4).
        // The same loop unmasks on the receive side (XOR is symmetric).
        uint8* mask = (uint8*)(&maskingKey);
        uint8* src = (uint8*)data;
        uint8* dst = (uint8*)(rawData + headerLength);
        for (uint64 n = 0; n < size; n++) {
            *dst = *src ^ mask[n % 4];
            dst++;
            src++;
        }
    }
    else {
        memcpy(rawData + headerLength, data, (uint32)size);
    }
    rawData[headerLength + payloadSize] = 0;
    status = COMPLETE;
    return true;
}
 
const char* WebsocketData::getRawData(uint64& size) {
    if (status != COMPLETE) {
        size = 0;
        return NULL;
    }
    size = headerLength + payloadSize;
    return rawData;
}
 
 
uint64 WebsocketData::getContentSize() {
    return contentSize;
}
 
uint64 WebsocketData::getPayloadSize() {
    return payloadSize;
}
 
 
bool WebsocketData::isComplete() {
    return (status == COMPLETE);
}
 
bool WebsocketData::processHeader(const char* buffer, uint64 size, bool &isInvalid) {
    // Decodes the RFC 6455 basic header (2 bytes) plus, depending on the 7-bit
    // length code, a 16-bit (code 126) or 64-bit (code 127) extended length in
    // network byte order, then the 4-byte masking key when the MASK bit is set.
    // Requires at least 4 bytes buffered even though a minimal unmasked frame
    // is only 2 — callers always have the masked client header available.
    // On a continuation frame (status != IDLE) the opcode is 0, so dataType is
    // only latched from the first fragment.
    isInvalid = false;
    if (!buffer || (size < 4))
        return false;
 
//  utils::WriteAFile("d:/ws.bin", buffer, size, true);
 
    time = GetTimeNow();
 
    uint8 b0 = *(const unsigned char*)buffer;
    uint8 b1 = *(const unsigned char*)(buffer + 1);
 
    isFinal = b0 & BHB0_FIN;
    uint8 opcode = b0 & BHB0_OPCODE;
 
    // if this is the first package
    if (status == IDLE) {
        if (opcode == 1)
            dataType = TEXT;
        else if (opcode == 2)
            dataType = BINARY;
        else if (opcode == 8)
            dataType = CLOSE;
        else if (opcode == 9)
            dataType = PING;
        else if (opcode == 10)
            dataType = PONG;
    }
 
    bool mask = b1 & BHB1_MASK;
    uint8 basic_length = b1 & BHB1_PAYLOAD;
    payloadSize = 0;
 
    uint32 mkOffset = basic_header_length;
    uint32 payloadOffset = basic_header_length;
 
    if (basic_length <= payload_size_basic) {
        payloadSize = basic_length;
    }
    else if (basic_length == payload_size_code_16bit) {
        uint16 h2 = *(const uint16*)(buffer + 2);
        payloadSize = ntohs(h2);
        mkOffset += 2;
        payloadOffset += 2;
        // get_extended_size(e);
    }
    else {
        uint64 h3 = *(const uint64*)(buffer + 2);
        payloadSize = utils::ntoh64(&h3);
        mkOffset += 8;
        payloadOffset += 8;
        // get_jumbo_size(e);
    }
 
    // if just a control message
    if (!payloadSize) {
        status = COMPLETE;
        return true;
    }
 
    maskingKey = 0;
    if (mask) {
        maskingKey = *(uint32*)(buffer + mkOffset);
        //maskingKey = 2112143814;
        payloadOffset += 4;
    }
 
    // printf("%s\n\n", utils::PrintBitFieldString(buffer, size, 8 * size, "ws").c_str());
 
/*  LogPrint(0, LOG_NETWORK, 3, "%s: [%u] (%u) size: %llu offset: %u    total size: %llu\n",
        isFinal ? "Final" : "Continue",
        opcode,
        maskingKey,
        payloadSize,
        payloadOffset,
        size
    );*/
 
    headerLength = payloadOffset;
    //contentSize = payloadSize;
 
    return true;
}
 
bool WebsocketData::processContent(const char* buffer, uint64 size) {
    // Unmasks (XOR with the 4-byte key; a no-op when maskingKey == 0, i.e.
    // unmasked server-to-client frames) and appends this fragment's payload.
    // Fragmented messages are reassembled here: each continuation reallocates
    // data to contentSize + payloadSize and copies the previous bytes over, so
    // the final buffer holds the whole logical message; status only becomes
    // COMPLETE on the FIN fragment.
    if (!buffer || (size != payloadSize))
        return false;
 
//  utils::WriteAFile("d:/ws_payload.bin", buffer, size, true);
 
    uint8* mask = (uint8*)(&maskingKey);
 
    uint8* src = (uint8*)buffer;
    uint8* dst;
 
    if (status == IDLE) {
        if (data)
            delete data;
        data = new char[(uint32)payloadSize + 1];
        dst = (uint8*)data;
    }
    else {
        char* newData = new char[(uint32)contentSize + (uint32)payloadSize + 1];
        memcpy(newData, data, (uint32)contentSize);
        delete data;
        data = newData;
        dst = (uint8*)(data+ (uint32)contentSize);
    }
 
    for (uint64 n = 0; n < payloadSize; n++) {
        *dst = *src ^ mask[n % 4];
        dst++;
        src++;
    }
    contentSize += payloadSize;
    data[contentSize] = 0;
    endReceiveTime = GetTimeNow();
    if (isFinal)
        status = COMPLETE;
    else
        status = INCOMPLETE;
    packages++;
 
    //LogPrint(0, LOG_NETWORK, 0, "Content received: %s\n",
    //  data
    //);
 
    if (isFinal) {
        if (packages == 1)
            LogPrint(0, LOG_NETWORK, 3, "Single Websocket package content received: %llu -> %llu\n", size, contentSize);
        else
            LogPrint(0, LOG_NETWORK, 3, "Last Websocket content package received: %llu -> %llu   (%u)\n", size, contentSize, packages);
    }
    else {
        if (packages == 1)
            LogPrint(0, LOG_NETWORK, 3, "First Websocket content received: %llu -> %llu\n", size, contentSize);
    //  else
    //      LogPrint(0, LOG_NETWORK, 0, "Next content received: %llu -> %llu\n", size, contentSize);
    }
 
//  utils::WriteAFile("d:/ws_content.bin", data, contentSize, true);
 
    return true;
}
 
 
 
const char* WebsocketData::getContent(uint64& size) {
    size = 0;
    if (!data || !payloadSize || (status != COMPLETE))
        return NULL;
    size = contentSize;
    return data;
}
 
 
 
 
 
 
 
 
 
 
 
 
TelnetLine::TelnetLine(uint64 source) {
    this->source = source;
    data = NULL;
    time = GetTimeNow();
    source = 0;
    size = 0;
    user = 0;
    cr[0] = 13;
    cr[1] = 10;
    cr[2] = 0;
}
 
TelnetLine::~TelnetLine() {
    if (data != NULL)
        delete(data);
    data=NULL;
}
 
bool TelnetLine::setCR(uint8 type) {
    switch(type) {
        default:
        case TELNET_WINDOWS:
            cr[0] = 13;
            cr[1] = 10;
            cr[2] = 0;
            break;
        case TELNET_UNIX:
            cr[0] = 10;
            cr[1] = 0;
            break;
    }
    return true;
}
 
uint32 TelnetLine::getSize() {
    if (!data)
        return 0;
    if (size)
        return size;
    else
        return (uint32)strlen(data);
}
 
bool TelnetLine::setLine(const char* buffer, bool addCR) {
    return setLine(buffer, (uint32)strlen(buffer), addCR);
}
 
bool TelnetLine::setLine(const char* buffer, uint32 len, bool addCR) {
    if (data)
        delete(data);
 
    data = new char[len+3];
    if (buffer && len > 0) {
        memcpy(data, buffer, len);
        if (addCR)
            memcpy(data+len, cr, 3);
        else
            data[len] = 0;
    }
    else
        data[len] = 0;
    return true;
}
 
bool TelnetLine::giveData(char* buffer, uint32 len) {
    if (data)
        delete(data);
    data = buffer;
    size = len;
    return true;
}
 
 
 
 
 
 
 
 
 
 
 
 
HTTPReply* HTTPReply::CreateErrorReply(uint8 type) {
    HTTPReply* reply = new HTTPReply((uint64)0);
    reply->type = type;
    return reply;
}
 
HTTPReply* HTTPReply::CreateAuthorizationReply(const char* realm) {
    HTTPReply* reply = new HTTPReply((uint64)0);
    reply->createAuthorizationReply(realm);
    return reply;
}
 
HTTPReply* HTTPReply::CreateWebsocketHTTPReply(const char* key, const char* version) {
    HTTPReply* reply = new HTTPReply((uint64)0);
    reply->createWebsocketHTTPReply(key, version);
    return reply;
}
 
 
HTTPReply::HTTPReply(uint64 source) {
    time = GetTimeNow();
    this->source = source;
    data = NULL;
    type = 0;
    headerLength = contentLength = 0;
    keepAlive = true;
}
 
HTTPReply::HTTPReply(HTTPReply* reply) {
    time = reply->time;
    source = reply->source;
    type = reply->type;
    headerLength = reply->headerLength;
    contentLength = reply->contentLength;
    keepAlive = reply->keepAlive;
    data = new char[headerLength+contentLength+1];
    memcpy(reply->data, data, headerLength+contentLength+1); // incl. string end
    entries = reply->entries;
    params = reply->params;
}
 
HTTPReply::~HTTPReply() {
    if (data != NULL)
        delete(data);
    data = NULL;
}
 
// Generating the data
 
bool HTTPReply::createOptionsResponse(uint8 status, uint64 time, const char* serverName, bool keepAlive, const char* origin, const char* operations) {
 
    //OPTIONS /cors HTTP/1.1
    //Origin: http://api.bob.com
    //Access-Control-Request-Method: PUT
    //Access-Control-Request-Headers: X-Custom-Header
    //Host: api.alice.com
    //Accept-Language: en-US
    //Connection: keep-alive
    //User-Agent: Mozilla/5.0...
    //
    //Preflight Response:
    //
    //Access-Control-Allow-Origin: http://api.bob.com
    //Access-Control-Allow-Methods: GET, POST, PUT
    //Access-Control-Allow-Headers: X-Custom-Header
    //Content-Type: text/html; charset=utf-8
 
    char timeString[1024];
    if (!GetHTTPTime(time, timeString, 1024))
        timeString[0] = 0;
 
    if (data)
        delete(data);
    data = new char[4096];
    snprintf(data, 4096,
"Access-Control-Allow-Origin: *\r\nAccess-Control-Allow-Methods: GET, PUT, POST, DELETE\r\nAccess-Control-Allow-Headers: accept, authorization, origin\r\n\r\n");
//"Access-Control-Allow-Origin: *\r\nAccess-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept\r\n\r\n");
 
//"%s\r\nDate: %s\r\n
//Server: %s\r\n
//Connection: %s\r\n
//"Access-Control-Allow-Origin: %s\r\n
//Access-Control-Allow-Methods: %s\r\n
//Access-Control-Allow-Headers: X-Custom-Header\r\n
//Content-Type: text/html\r\n\r\n",
//      //HTTP_Status[status], timeString, serverName,
//      //keepAlive ? "Keep-Alive" : "close",
//      origin, operations);
 
    headerLength = (uint32)strlen(data);
    contentLength = 0;
    return true;
}
 
 
bool HTTPReply::createErrorPage(uint8 status, uint64 time, const char* serverName, bool keepAlive) {
 
    char timeString[1024];
    if (!GetHTTPTime(time, timeString, 1024))
        timeString[0] = 0;
 
    if (data)
        delete(data);
    data = new char[4096];
    snprintf(data, 4096,
        "%s\r\nDate: %s\r\nServer: %s\r\n"
        "X-Frame-Options: SAMEORIGIN\r\n"
        "Connection: %s\r\n\r\n",
        HTTP_Status[status], timeString, serverName,
        keepAlive ? "Keep-Alive" : "close");
 
    headerLength = (uint32)strlen(data);
    contentLength = 0;
    return true;
}
 
bool HTTPReply::createAuthorizationReply(const char* realm) {
 
    char timeString[1024];
    if (!GetHTTPTime(time, timeString, 1024))
        timeString[0] = 0;
 
    if (data)
        delete(data);
    data = new char[4096];
    snprintf(data, 4096,
        "%s\r\nWWW-Authenticate: Basic realm=\"%s\"\r\nContent-Length: 0\r\n\r\n",
        HTTP_Status[HTTP_UNAUTHORIZED], realm);
 
    headerLength = (uint32)strlen(data);
    contentLength = 0;
    return true;
}
 
bool HTTPReply::createWebsocketHTTPReply(const char* key, const char* version) {
    // RFC 6455 §4.2.2 handshake: the server proves it understood the upgrade by
    // concatenating the client's Sec-WebSocket-Key with the fixed magic GUID
    // 258EAFA5-E914-47DA-95CA-C5AB0DC85B11, SHA-1 hashing the result and
    // returning it base64-encoded as Sec-WebSocket-Accept in a 101 reply.
 
    if (!key || !strlen(key))
        return false;
 
    std::string magicString = utils::StringFormat("%s258EAFA5-E914-47DA-95CA-C5AB0DC85B11", key);
    hash::SHA1 sha1;
    sha1.reset();
    sha1.add(magicString.c_str(), magicString.size());
    std::string rawMagic = sha1.getHashRaw();
 
    std::string acceptKey = base64_encode(rawMagic);
 
    if (data)
        delete(data);
    data = new char[4096];
    snprintf(data, 4096,
        "%s\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: %s\r\n\r\n",
        HTTP_Status[HTTP_SWITCH_PROTOCOL], acceptKey.c_str());
 
    headerLength = (uint32)strlen(data);
    contentLength = 0;
    return true;
}
 
bool HTTPReply::createPage(uint8 status, uint64 time, const char* serverName,
    uint64 lastMod, bool keepAlive, bool cache, const char* contentType,
    const char* content, uint32 contentSize, const char* additionalHeaderEntries) {
 
    char timeString[1024];
    if (!GetHTTPTime(time, timeString, 1024))
        timeString[0] = 0;
 
    contentLength = (contentSize > 0) ? contentSize : (uint32)strlen(content);
 
    if (data)
        delete(data);
    data = new char[4096 + contentLength];
    if (additionalHeaderEntries && strlen(additionalHeaderEntries)) {
        snprintf(data, 4096 + contentLength,
            "%s\r\nDate: %s\r\nServer: %s\r\n"
            "X-Frame-Options: SAMEORIGIN\r\n"
            "Last-Modified: %s\r\nContent-Length: %u\r\n"
            "Connection: %s\r\nContent-Type: %s\r\nCache-Control: %s\r\n"
            "%s\r\n\r\n",
            HTTP_Status[status], timeString, serverName, timeString, contentLength,
            keepAlive ? "Keep-Alive" : "close", contentType, cache ? "Public" : "No-Cache",
            additionalHeaderEntries);
    }
    else {
        snprintf(data, 4096 + contentLength,
            "%s\r\nDate: %s\r\nServer: %s\r\n"
            "X-Frame-Options: SAMEORIGIN\r\n"
            "Last-Modified: %s\r\nContent-Length: %u\r\n"
            "Connection: %s\r\nContent-Type: %s\r\nCache-Control: %s\r\n\r\n",
            HTTP_Status[status], timeString, serverName, timeString, contentLength,
            keepAlive ? "Keep-Alive" : "close", contentType, cache ? "Public" : "No-Cache");
    }
 
    headerLength = (uint32)strlen(data);
    memcpy(data+headerLength, content, contentLength);
    data[headerLength + contentLength] = 0;
 
    return true;
}
 
bool HTTPReply::createFromFile(uint64 time, const char* serverName,
    uint64 ifLastMod, bool keepAlive, bool cache,
    const char* filename) {
 
    std::string actualFilename = filename;
    const char* question = strstr(filename, "?");
    if (question)
        actualFilename = std::string(filename, question - filename);
 
    uint64 lastModifiedTime = 0;
    if (ifLastMod) {
        utils::FileDetails fileInfo = utils::GetFileDetails(actualFilename.c_str());
        if (fileInfo.doesExist && !fileInfo.isDirectory && (fileInfo.lastModifyTime < ifLastMod) )
            return createErrorPage(HTTP_USE_LOCAL_COPY, time, serverName, keepAlive);
    }
 
    char* filedata = NULL;
    uint32 datasize = 0;
    std::string html;
    bool isBinary = false;
    char* exttype = new char[1024];
    if (!html::GetMimeType(exttype, actualFilename.c_str(), 1024, isBinary)) {
        html = utils::StringFormat("Unsupported mime type for file '%s'...", actualFilename.c_str());
    }
    else if (!isBinary) {
        if (!(html = utils::ReadAFileString(actualFilename.c_str())).size()) {
            //html = utils::StringFormat("Could read file '%s'", actualFilename.c_str());
        }
    }
    else {
        if ( !(filedata = utils::ReadAFile(actualFilename.c_str(), datasize, true))) {
            //html = utils::StringFormat("Could read file '%s'", actualFilename.c_str());
        }
    }
 
    if (filedata)
        createPage(HTTP_OK, GetTimeNow(), serverName, 0, keepAlive, cache, exttype, filedata, datasize);
    else if (html.length())
        createPage(HTTP_OK, GetTimeNow(), serverName, 0, keepAlive, cache, exttype, html.c_str());
    else {
        html = "<html><head><title>404 Not Found</title></head><body><h1>Not Found</h1><p>The requested URL was not found on this server.</p></body></html>";
        createPage(HTTP_FILE_NOT_FOUND, GetTimeNow(), serverName, 0, keepAlive, cache, "text/html", html.c_str());
    }
 
    delete [] filedata;
    delete [] exttype;
 
    return true;
}
 
bool HTTPReply::isWebsocketUpgrade() {
    if (type != HTTP_SWITCH_PROTOCOL)
        return false;
 
    const char* connection = this->getHeaderEntry("Connection");
    if (!connection || stricmp(connection, "Upgrade"))
        return false;
 
    const char* upgrade = this->getHeaderEntry("Upgrade");
    if (!upgrade || stricmp(upgrade, "websocket"))
        return false;
 
    return true;
}
 
 
// Reading it in from a socket
bool HTTPReply::processHeader(const char* buffer, uint32 size, bool &isInvalid) {
    isInvalid = false;
    if (!buffer || (size < 5))
        return false;
 
    // Find end of header
    uint32 n;
    for (n = 3; n < size; n++) {
        if (buffer[n] == 10) {
            if ( (size-n >= 2) && (buffer[n+1] == 10) ) {
                // Just using LF, no CR
                headerLength = n+2;
                break;
            }
            else if ( (size-n >= 3) && (buffer[n+1] == 13) && (buffer[n+2] == 10) && (buffer[n-1] == 13) ) {
                // Using CRLF
                headerLength = n+3;
                break;
            }
        }
    }
 
    if (headerLength == 0) {
        isInvalid = true;
        return false;
    }
 
    uint32 tempContentLength = 4096;
    data = new char[headerLength+tempContentLength+1];
    memcpy(data, buffer, headerLength);
    data[headerLength] = 0; // temp string end
 
    // Now process header entries
    std::string key, val, uri;
    const char* s = data, *t;
    n = 0;
    uint32 crSize = 0;
    uint32 left = headerLength;
    while (n < headerLength) {
        if (!utils::GetNextLineEnd(s, left, n, crSize))
            break;
        // Process line
        //if (t = strchr(s, ' ')) {
        if ((t = strchr(s, ' ')) && (t - s < (int32)n)) {
                t -= 1;
            if (*t != ':') {
                t++;
                key.assign(s, t-s);
            }
            else {
                key.assign(s, t-s);
                t++;
            }
            while (*t == 32) t++;
            if (*t > 32)
                val.assign(t, n+s-t);
            else
                val = "";
        }
        else {
            key.assign(s, n);
            val = "";
        }
        // Record entry
        if (s == data) {
            entries["http_protocol"] = key;
            if (val.find("200") != val.npos)
                type = HTTP_OK;
            else if (val.find("101") != val.npos)
                type = HTTP_SWITCH_PROTOCOL;
            else if (val.find("301") != val.npos)
                type = HTTP_MOVED_PERMANENTLY;
            else if (val.find("304") != val.npos)
                type = HTTP_USE_LOCAL_COPY;
            else if (val.find("403") != val.npos)
                type = HTTP_ACCESS_DENIED;
            else if (val.find("404") != val.npos)
                type = HTTP_FILE_NOT_FOUND;
            else if (val.find("500") != val.npos)
                type = HTTP_INTERNAL_ERROR;
            else
                type = atoi(val.c_str());
        }
        else {
            entries[key] = val;
            // Check entry
            if (stricmp(key.c_str(), "content-length") == 0)
                contentLength = atoi(val.c_str());
            else if (stricmp(key.c_str(), "date") == 0)
                time = GetTimeFromString(val.c_str());
            else if (stricmp(key.c_str(), "connection") == 0)
                keepAlive = (stricmp(val.c_str(), "close") != 0);
        }
        // Next line
        s += n + crSize;
        left -= n + crSize;
    }
 
    if (contentLength > tempContentLength) {
        delete(data);
        data = new char[headerLength+contentLength+1];
        memcpy(data, buffer, headerLength);
        data[headerLength] = 0; // temp string end
        data[headerLength+contentLength] = 0;
    }
 
    return true;
}
 
bool HTTPReply::processContent(const char* buffer, uint32 size) {
    if (!buffer || (size != contentLength))
        return false;
    memcpy(data+headerLength, buffer, size);
    data[headerLength+contentLength] = 0;
    return true;
}
 
const char* HTTPReply::getContent(uint32& size) {
    size = contentLength;
    return data+headerLength;
}
 
const char* HTTPReply::getRawContent(uint32& size) {
    size = contentLength + headerLength;
    return data;
}
 
const char* HTTPReply::getHeaderEntry(const char* entry) {
    std::map<std::string, std::string>::iterator it = entries.find(entry);
    if (it != entries.end())
        return it->second.c_str();
    else
        return NULL;
}
 
const char* HTTPReply::getProtocol() {
    return getHeaderEntry("http_protocol");
}
 
uint32 HTTPReply::getSize() {
    if (data)
        return headerLength + contentLength;
    else
        return 0;
}
 

// HTTP Server Protocol
 
bool HTTPProtocol::CheckBufferForCompatibility(const char* buffer, uint32 length) {
    if (length < 4)
        return false;
    return ( utils::stristr(buffer, "GET ") == buffer ||
        utils::stristr(buffer, "PUT ") == buffer ||
        utils::stristr(buffer, "POST ") == buffer ||
        utils::stristr(buffer, "DELETE ") == buffer ||
        utils::stristr(buffer, "OPTIONS ") == buffer ||
        utils::stristr(buffer, "HEAD ") == buffer );
}
 
bool HTTPProtocol::InitialiseConversation(NetworkConnection* con) {
    return false;
}
 
bool HTTPProtocol::SendHTTPReply(NetworkConnection* con, HTTPReply* reply) {
    if (!con || !reply)
        return false;
    //return con->send((char*)reply->data, reply->getSize());
    if (con->send((char*)reply->data, reply->getSize())) {
        LogPrint(0, LOG_NETWORK, 4, "Replying to HTTP request with %u bytes", reply->getSize());
        // printf("--- HTTP sent reply (%u)...\n", reply->contentLength);
        return true;
    }
    else {
        LogPrint(0, LOG_NETWORK, 2, "Failed to send reply to HTTP request with %u bytes", reply->getSize());
        // printf("--- HTTP failed to send reply (%u)...\n", reply->contentLength);
        return false;
    }
 
}
 
HTTPReply* HTTPProtocol::ReceiveHTTPReply(NetworkConnection* con, uint32 timeout) {
 
    if (!con || !con->waitForDataToRead(timeout))
        return NULL;
        // return HTTPReply::CreateErrorReply(HTTP_SERVER_UNAVAILABLE);
 
    uint32 maxSize = 4096, size;
    char* buffer = new char[maxSize];
 
    uint64 start = GetTimeNow();
    uint32 wait = 0;
 
    bool isInvalid = false;
    HTTPReply* reply = NULL;
    while (true) {
        if (!con->receiveAvailable(buffer, size, maxSize, wait, true)) {
            delete [] buffer;
            return HTTPReply::CreateErrorReply(HTTP_SERVER_NOREPLY);
        }
        if (size > 0) {
            reply = new HTTPReply(con->getRemoteAddress());
            if (reply->processHeader(buffer, size, isInvalid))
                break;
            delete(reply);
            reply = NULL;
            if (isInvalid) {
                // ##############
            }
        }
        if (GetTimeAgeMS(start) < (int32)timeout) {
            delete [] buffer;
            return HTTPReply::CreateErrorReply(HTTP_SERVER_NOREPLY);
        }
        wait = 20;
    }
 
    // Now we know how much to actually expect
    // First read the header proper and ignore
    if (!con->discard(reply->headerLength)) {
        delete [] buffer;
        delete(reply);
        return HTTPReply::CreateErrorReply(HTTP_SERVER_NOREPLY);
    }
 
//  printf("Receiving HTTPReply content (%u)...\n\n", reply->contentLength);
    // Then read the rest of the content
    if (reply->contentLength) {
        if (reply->contentLength > maxSize - reply->headerLength) {
            delete [] buffer;
            buffer = new char[reply->contentLength];
            // We have to read the rest, otherwise we will mess up the connection
            if (!con->receive(buffer, reply->contentLength, 3000)) {
                printf("Error receiving HTTPReply content (%u)...\n\n", reply->contentLength);
                delete [] buffer;
                delete(reply);
                return HTTPReply::CreateErrorReply(HTTP_SERVER_MALFORMED_REPLY);
            }
 
            if (!reply->processContent(buffer, reply->contentLength)) {
                delete [] buffer;
                delete(reply);
                return HTTPReply::CreateErrorReply(HTTP_SERVER_MALFORMED_REPLY);
            }
        }
        else {
            // We have to read the rest, otherwise we will mess up the connection
            if (!con->receive(buffer, reply->contentLength, 3000)) {
                printf("Error receiving HTTPReply content (%u)...\n\n", reply->contentLength);
                delete [] buffer;
                delete(reply);
                return HTTPReply::CreateErrorReply(HTTP_SERVER_MALFORMED_REPLY);
            }
 
            if (!reply->processContent(buffer, reply->contentLength)) {
                delete [] buffer;
                delete(reply);
                return HTTPReply::CreateErrorReply(HTTP_SERVER_MALFORMED_REPLY);
            }
        }
    }
 
//  printf("Got HTTPReply...\n\n");
    delete [] buffer;
    return reply;
}
 
bool HTTPProtocol::SendHTTPRequest(NetworkConnection* con, HTTPRequest* req) {
    if (!con || !req)
        return false;
    return con->send((char*)req->data, req->getSize());
}
 
HTTPRequest* HTTPProtocol::ReceiveHTTPRequest(NetworkConnection* con, uint32 timeout) {
 
    // ##########################
 
    if (!con || !con->waitForDataToRead(timeout)) {
        return NULL;
    }
 
    uint32 maxSize = 4096, size;
    char* buffer = new char[maxSize];
 
    uint64 start = GetTimeNow();
    uint64 startReceive = 0;
    uint32 wait = 0;
 
    bool isInvalid;
    HTTPRequest* req = NULL;
    while (true) {
        if (!con->receiveAvailable(buffer, size, maxSize, wait, true)) {
            delete [] buffer;
            return NULL;
        }
        if (size > 0) {
            LogPrint(0, LOG_NETWORK, 5, "Started receiving incoming HTTP request, size so far %u", size);
            if (!startReceive)
                startReceive = GetTimeNow();
            req = new HTTPRequest(con->getRemoteAddress(), startReceive);
            //utils::PrintBinary(buffer, size, false, "HTTP: ");
            if (req->processHeader(buffer, size, isInvalid)) {
                //printf("HTTP Done\n");
                break;
            }
            else if (isInvalid) {
                LogPrint(0, LOG_NETWORK, 2, "Received invalid incoming HTTP request, size %u", size);
                //utils::WriteAFile("d:\\http_invalid.bin", buffer, size, true);
                //DataMessage* test = new DataMessage(buffer);
                delete [] buffer;
                con->discard(size);
                return req;
            }
            else if (size == maxSize) {
                LogPrint(0, LOG_NETWORK, 2, "Received incoming HTTP request, buffer needs resizing %u", size);
                // maxSize isn't big enough to satisfy processHeader
                //printf("HTTP MaxSize, waiting for more...\n");
                //utils::PrintBinary(buffer, size, false, "HTTP MaxSize: ");
                //utils::WriteAFile("d:\\http_maxsize.bin", buffer, size, true);
                utils::Sleep(10);
            }
            else {
                LogPrint(0, LOG_NETWORK, 5, "Received incoming HTTP request, got %u, waiting for more...", size);
                //utils::PrintBinary(buffer, size, false, "HTTP WaitMore: ");
                //printf("HTTP waiting for more...\n");
                //utils::WriteAFile("d:\\http_notenough.bin", buffer, size, true);
                utils::Sleep(10);
            }
            delete(req);
            req = NULL;
        }
        if (GetTimeAgeMS(start) < (int32)timeout) {
            delete [] buffer;
            return NULL;
        }
        wait = 20;
    }
 
    // Now we know how much to actually expect
    // First read the header proper and ignore
    if (!con->discard(req->headerLength)) {
        delete [] buffer;
        delete(req);
        return NULL;
    }
    // Then read the rest of the content
    if (req->contentLength) {
        if (req->contentLength > maxSize) {
            delete [] buffer;
            buffer = new char[req->contentLength];
        }
        // We have to read the rest, otherwise we will mess up the connection
        if (!con->receive(buffer, req->contentLength, 30000)) {
            LogPrint(0, LOG_NETWORK, 2, "Received HTTP header of %u, couldn't read the content of size %u", req->headerLength, req->contentLength);
            delete [] buffer;
            delete(req);
            return NULL;
        }
 
        if (!req->processContent(buffer, req->contentLength)) {
            LogPrint(0, LOG_NETWORK, 2, "Received HTTP header of %u, content of size %u, failed processing the content buffer", req->headerLength, req->contentLength);
            delete [] buffer;
            delete(req);
            return NULL;
        }
    }
 
    delete [] buffer;
    return req;
}
 
 
 
WebsocketData* HTTPProtocol::ReceiveWebsocketData(NetworkConnection* con, uint32 timeout) {
 
    if (!con || !con->waitForDataToRead(timeout)) {
        return NULL;
    }
 
    uint32 maxSize = 4096, size;
    char* buffer = new char[maxSize];
 
    uint64 start = GetTimeNow();
    uint64 startReceive = 0;
    uint32 wait = 0;
    uint64 remoteAddress;
    bool isInvalid;
    WebsocketData* wsData = NULL;
    while (!wsData || !wsData->isComplete()) {
        while (true) {
            if (!wsData && !con->receiveAvailable(buffer, size, maxSize, wait, true)) {
                delete[] buffer;
                return NULL;
            }
            else
                con->receiveAvailable(buffer, size, maxSize, wait, true);
            if (size > 0) {
                //utils::WriteAFile("d:/c/ws.bin", buffer, size, true);
                if (!wsData) {
                    if (!startReceive)
                        startReceive = GetTimeNow();
                    wsData = new WebsocketData(con->getRemoteAddress(), startReceive);
                    //utils::PrintBinary(buffer, size, false, "HTTP: ");
                }
 
                if (wsData->processHeader(buffer, size, isInvalid)) {
                    //printf("HTTP Done\n");
                    break;
                }
                else if (isInvalid) {
                    remoteAddress = con->getRemoteAddress();
                    LogPrint(0, LOG_NETWORK, 1, "Invalid data received on Websocket from %u.%u.%u.%u", GETIPADDRESSQUAD(remoteAddress));
                    //utils::WriteAFile("d:\\http_invalid.bin", buffer, size, true);
                    //DataMessage* test = new DataMessage(buffer);
                    delete[] buffer;
                    con->discard(size);
                    return wsData;
                }
                else if (size == maxSize) {
                    // maxSize isn't big enough to satisfy processHeader
                    //printf("HTTP MaxSize, waiting for more...\n");
                    //utils::PrintBinary(buffer, size, false, "HTTP MaxSize: ");
                    //utils::WriteAFile("d:\\http_maxsize.bin", buffer, size, true);
                    utils::Sleep(10);
                }
                else {
                    //utils::PrintBinary(buffer, size, false, "HTTP WaitMore: ");
                    //printf("HTTP waiting for more...\n");
                    //utils::WriteAFile("d:\\http_notenough.bin", buffer, size, true);
                    utils::Sleep(10);
                }
                delete(wsData);
                wsData = NULL;
            }
            if (!wsData && (GetTimeAgeMS(start) < (int32)timeout)) {
                remoteAddress = con->getRemoteAddress();
                LogPrint(0, LOG_NETWORK, 1, "Timeout receiving data on Websocket from %u.%u.%u.%u", GETIPADDRESSQUAD(remoteAddress));
                delete[] buffer;
                return NULL;
            }
            wait = 20;
        }
 
        // Now we know how much to actually expect
        // First read the header proper and ignore
        if (!con->discard(wsData->headerLength)) {
            delete[] buffer;
            delete(wsData);
            return NULL;
        }
        size -= wsData->headerLength;
        // Then read the rest of the content
        if (wsData->payloadSize) {
            if (wsData->payloadSize > maxSize) {
                delete[] buffer;
                buffer = new char[(uint32)(wsData->payloadSize)];
            }
            // We have to read the rest, otherwise we will mess up the connection
            if (!con->receive(buffer, (uint32)wsData->payloadSize, 30000)) {
                remoteAddress = con->getRemoteAddress();
                LogPrint(0, LOG_NETWORK, 1, "Unable to receive payload data on Websocket from %u.%u.%u.%u", GETIPADDRESSQUAD(remoteAddress));
                delete[] buffer;
                delete(wsData);
                return NULL;
            }
 
            if (!wsData->processContent(buffer, wsData->payloadSize)) {
                remoteAddress = con->getRemoteAddress();
                LogPrint(0, LOG_NETWORK, 1, "Unable to process payload data on Websocket from %u.%u.%u.%u", GETIPADDRESSQUAD(remoteAddress));
                delete[] buffer;
                delete(wsData);
                return NULL;
            }
            size -= (uint32)(wsData->payloadSize);
        }
        if (wsData->isComplete()) {
            break;
        }
    }
 
    delete[] buffer;
    return wsData;
}
 
bool HTTPProtocol::SendWebsocketData(NetworkConnection* con, WebsocketData* wsData) {
    if (!con || !wsData)
        return false;
    uint64 size = 0;
    const char* data = wsData->getRawData(size);
    if (!data || !size)
        return false;
    //return con->send((char*)reply->data, reply->getSize());
    if (con->send(data, (uint32)size)) {
        // printf("--- HTTP sent reply (%u)...\n", reply->contentLength);
        return true;
    }
    else {
        // printf("--- HTTP failed to send reply (%u)...\n", reply->contentLength);
        return false;
    }
 
}
 
 
 
 

// HTTP Client Protocol
 
bool HTTPClientProtocol::CheckBufferForCompatibility(const char* buffer, uint32 length) {
    return false;
}
 
bool HTTPClientProtocol::InitialiseConversation(NetworkConnection* con) {
    return false;
}
 
bool HTTPClientProtocol::SendHTTPReply(NetworkConnection* con, HTTPReply* reply) {
    return false;
}
 
HTTPReply* HTTPClientProtocol::ReceiveHTTPReply(NetworkConnection* con, uint32 timeout) {
    return NULL;
}
 
bool HTTPClientProtocol::SendHTTPRequest(NetworkConnection* con, HTTPRequest* req) {
    return false;
}
 
HTTPRequest* HTTPClientProtocol::ReceiveHTTPRequest(NetworkConnection* con, uint32 timeout) {
    return NULL;
}
 
 

// Message Protocol
 
bool MessageProtocol::CheckBufferForCompatibility(const char* buffer, uint32 length) {
    if (length < 2*sizeof(uint32))
        return false;
    return (GetObjID(buffer) == DATAMESSAGEID);
}
 
bool MessageProtocol::InitialiseConversation(NetworkConnection* con) {
    return true;
}
 
bool MessageProtocol::SendMessage(NetworkConnection* con, DataMessage* msg, uint64 receiver) {
    if (!con || !msg)
        return false;
//  uint64 addr = con->getRemoteAddress();
//  LogPrint(0,0,0,"Protocol sending msg %llu to %u.%u.%u.%u:%u", msg->getType(), GETIPADDRESSQUAD(addr), GETIPPORT(addr));
//  LogPrint(0,0,0,"Protocol sending msg: %u", msg->getType()[15]);
    return con->send((char*)msg->data, msg->getSize(), receiver);
}
 
DataMessage* MessageProtocol::ReceiveMessage(NetworkConnection* con, uint32 timeout) {
//  uint64 t = GetTimeNow();
    if (!con) {
//  if (!con || !con->waitForDataToRead(timeout)) {
//      printf("Failed ReceiveMessage:      %ld\n", GetTimeAgeMS(t));
        return NULL;
    }
 
    // Peek just the header (size + object id) into a small stack buffer so the
    // body allocation can be sized exactly once. The previous code malloc'd a 1KB
    // scratch buffer for every message and then freed and re-malloc'd it for any
    // message bigger than 1KB - two allocations plus a free per large message.
    char header[2*sizeof(uint32)];
    if (!con->receive(header, sizeof(header), timeout, true))
        return NULL;
    if (GetObjID(header) != DATAMESSAGEID)
        return NULL;
    uint32 size = *(uint32*)header;
    if (size < sizeof(header))
        return NULL;   // implausible size for a DataMessage header
 
    char* buffer = (char*) malloc(size);
    if (!buffer)
        return NULL;
    if (!con->receive(buffer, size, timeout)) {
        free(buffer);
        return NULL;
    }
    // DataMessage adopts the buffer (copy=false) and frees it on destruction.
    DataMessage* msg = new DataMessage(buffer);
//  uint64 addr = con->getRemoteAddress();
//  LogPrint(0,0,0,"Protocol received msg %llu from %u.%u.%u.%u:%u", msg->getType(), GETIPADDRESSQUAD(addr), GETIPPORT(addr));
    //printf("Protocol received msg: %u.%u.%u (%p)...\n", msg->getType()[0], msg->getType()[1], msg->getType()[2], msg);
    return msg;
}
 

// Telnet Protocol
 
bool TelnetProtocol::CheckBufferForCompatibility(const char* buffer, uint32 length) {
    if (length < 1)
        return false;
    return ((buffer[0] == 13) || (buffer[0] == 10));
}
 
bool TelnetProtocol::InitialiseConversation(NetworkConnection* con) {
    return true;
}
 
bool TelnetProtocol::SendTelnetLine(NetworkConnection* con, TelnetLine* line) {
    if (!con || !line)
        return false;
    return con->send((char*)line->data, line->getSize());
}
 
TelnetLine* TelnetProtocol::ReceiveTelnetLine(NetworkConnection* con, uint32 timeout) {
    if (!con || !con->waitForDataToRead(timeout))
        return NULL;
 
    uint32 maxSize = 1024, size;
    char* buffer = new char[maxSize];
    TelnetLine* line = NULL;
 
    uint64 start = GetTimeNow();
    uint32 wait = 0, n;
 
    while (GetTimeAgeMS(start) < (int32)timeout) {
        if (!con->receiveAvailable(buffer, size, maxSize, wait, true)) {
            delete [] buffer;
            return NULL;
        }
        for ( n=0; n<size; n++) {
            if (buffer[n] == 13) {
                line = new TelnetLine(con->getRemoteAddress());
                line->setCR(TELNET_WINDOWS);
                line->setLine(buffer, n);
                if ( (n < size-1) && (buffer[n+1] == 10) )
                    n++;
                con->discard(n+1);
                delete [] buffer;
            //  printf("\n");
                return line;
            }
            else if (buffer[n] == 10) {
                line = new TelnetLine(con->getRemoteAddress());
                line->setCR(TELNET_UNIX);
                line->setLine(buffer, n);
                if ( (n < size-1) && (buffer[n+1] == 13) )
                    n++;
                con->discard(n+1);
                delete [] buffer;
            //  printf("\n");
                return line;
            }
            //else
            //  printf("[%c]", buffer[n]);
        }
        if (size == maxSize) {
            delete [] buffer;
            maxSize *= 4;
            buffer = new char[maxSize];
            wait = 0;
        }
        else
            wait = 20;
    }
    delete [] buffer;
    return NULL;
}
 
TelnetLine* TelnetProtocol::ReceiveTelnetLine(NetworkConnection* con, uint32 timeout, uint32 size) {
    if (!con || !con->waitForDataToRead(timeout))
        return NULL;
 
    char* buffer = new char[size];
 
    if (!con->receive(buffer, size, timeout)) {
        delete [] buffer;
        return NULL;
    }
 
    TelnetLine* reply = new TelnetLine(con->getRemoteAddress());
    reply->data = buffer;
    reply->size = size;
    return reply;
}
 
 
 
 
 
 
JSONM::JSONM() {
    jmData = NULL;
    jmSize = mOffset = 0;
}
 
JSONM::JSONM(const char* data, uint64 size) {
    jmSize = size;
    if (jmSize) {
        jmData = new char[(uint32)jmSize];
        memcpy(jmData, data, (uint32)jmSize);
        mOffset = strlen(data) + 1;
        extractMultipartInfo();
    }
    else
        jmData = NULL;
}
 
JSONM::JSONM(const char* json) {
    jmData = NULL;
    jmSize = 0;
    if (json)
        setJSON(json);
    else
        setJSON("{}");
}
 
JSONM::~JSONM() {
    mutex.enter(1000);
    if (jmData)
        delete[] jmData;
    jmData = NULL;
    jmSize = 0;
    mOffset = 0;
}
 
uint64 JSONM::getSize() {
    if (!mutex.enter(1000)) return 0;
    uint64 s = jmSize;
    mutex.leave();
    return s;
}
uint32 JSONM::getCount() {
    if (!mutex.enter(1000)) return 0;
    uint32 s = (uint32)entries.size();
    mutex.leave();
    return s;
}
 
bool JSONM::isValid() {
    if (!jmData || !jmSize)
        return false;
    if (!strchr(jmData, '{') || !utils::laststrstr(jmData, "}"))
        return false;
    // Check that the sizes add up to the correct full size
    uint64 jsonLength = strlen(jmData);
    uint64 multipartLength = 0;
    std::map<uint32, JSONMEntry>::iterator i = entries.begin(), e = entries.end();
    while (i != e) {
        multipartLength += i->second.size;
        i++;
    }
    if (jsonLength + 1 + multipartLength != jmSize) {
        LogPrint(0, LOG_NETWORK, 0, "Invalid JSONM structure: JSON(%llu) + 1 + multiparts(%llu) != %llu", jsonLength, multipartLength, jmSize);
        return false;
    }
    return true;
}
 
std::string JSONM::getJSON() {
    if (!jmData || !jmSize)
        return "";
    if (!strchr(jmData, '{') || !utils::laststrstr(jmData, "}"))
        return "";
    if (strlen(jmData) > jmSize)
        return std::string(jmData, jmSize);
    return jmData;
}
 
const char* JSONM::getData(const char* name, uint64& size) {
    if (!mutex.enter(1000)) return NULL;
    size = 0;
    if (!name || !jmData || !jmSize) {
        mutex.leave();
        return NULL;
    }
    std::map<uint32, JSONMEntry>::iterator i = entries.begin(), e = entries.end();
    while (i != e) {
        if (stricmp(i->second.name.c_str(), name) == 0) {
            if (mOffset + i->second.offset + i->second.size > jmSize) {
                LogPrint(0, LOG_NETWORK, 0, "Invalid JSONM structure: Entry '%s' offset %llu size %llu exceeds the JSONM size %llu",
                    name, i->second.offset, i->second.size, jmSize);
                mutex.leave();
                return NULL;
            }
            size = i->second.size;
            mutex.leave();
            return jmData + mOffset + i->second.offset;
        }
        i++;
    }
    mutex.leave();
    return NULL;
}
 
std::string JSONM::getDataType(const char* name) {
    if (!mutex.enter(1000)) return "";
    if (!name || !jmData || !jmSize) {
        mutex.leave();
        return NULL;
    }
    std::map<uint32, JSONMEntry>::iterator i = entries.begin(), e = entries.end();
    while (i != e) {
        if (stricmp(i->second.name.c_str(), name) == 0) {
            mutex.leave();
            return i->second.type;
        }
        i++;
    }
    mutex.leave();
    return "";
}
 
std::string JSONM::getDataType(uint32 chunk) {
    if (!mutex.enter(1000)) return NULL;
    if (!jmData || !jmSize || !chunk)
        return "";
 
    std::map<uint32, JSONMEntry>::iterator i = entries.find(chunk);
    if (i == entries.end()) {
        mutex.leave();
        return "";
    }
 
    mutex.leave();
    return i->second.type;
}
 
std::string JSONM::getDataName(uint32 chunk) {
    if (!mutex.enter(1000)) return NULL;
    if (!jmData || !jmSize || !chunk) {
        mutex.leave();
        return "";
    }
 
    std::map<uint32, JSONMEntry>::iterator i = entries.find(chunk);
    if (i == entries.end()) {
        mutex.leave();
        return "";
    }
 
    mutex.leave();
    return i->second.name;
}
 
 
 
const char* JSONM::getData(uint32 chunk, uint64& size) {
    if (!mutex.enter(1000)) return NULL;
    size = 0;
    if (!jmData || !jmSize) {
        mutex.leave();
        return NULL;
    }
 
    std::map<uint32, JSONMEntry>::iterator i = entries.find(chunk);
    if (i == entries.end()) {
        mutex.leave();
        return NULL;
    }
 
    if (mOffset + i->second.offset + i->second.size > jmSize) {
        LogPrint(0, LOG_NETWORK, 0, "Invalid JSONM structure: Entry %u '%s' offset %llu size %llu exceeds the JSONM size %llu",
            chunk, i->second.name.c_str(), i->second.offset, i->second.size, jmSize);
        mutex.leave();
        return NULL;
    }
 
    size = i->second.size;
    mutex.leave();
    return jmData + mOffset + i->second.offset;
}
 
bool JSONM::setRawJSON(const char* json) {
    if (!json)
        return false;
    if (!jmData) {
        jmSize = mOffset = strlen(json) + 1;
        jmData = new char[(uint32)jmSize];
        utils::strcpyavail(jmData, json, (uint32)jmSize, true);
        return true;
    }
    // else we have JSON and possibly data already
    uint64 newMOffset = strlen(json) + 1;
    uint64 newSize = newMOffset + (jmSize - mOffset);
    char* newData = new char[(uint32)newSize];
    utils::strcpyavail(newData, json, (uint32)newMOffset, true);
    // copy in old data
    memcpy(newData + (uint32)newMOffset, jmData + (uint32)mOffset, (uint32)(jmSize - mOffset));
    delete[] jmData;
    jmData = newData;
    mOffset = newMOffset;
    jmSize = newSize;
    return true;
}
 
bool JSONM::setJSON(const char* json) {
    if (!json)
        return false;
    if (!mutex.enter(1000)) return false;
    if (setRawJSON(json))
        updateMultipartJSON();
    mutex.leave();
    return true;
}
 
uint32 JSONM::addData(const char* name, const char* data, uint64 size, const char* type) {
    if (!mutex.enter(1000)) return 0;
    // check for name in use
    uint64 s;
    if (getData(name, s)) {
        mutex.leave();
        return 0;
    }
    if (!jmData)
        setJSON("{}");
    JSONMEntry entry;
    entry.chunk = getCount() + 1;
    entry.name = name;
    entry.size = size;
    entry.type = type;
    entry.offset = jmSize - this->mOffset;
    uint64 newSize = jmSize + size;
    char* newData = new char[(uint32)newSize];
    memcpy(newData, jmData, (uint32)jmSize);
    memcpy(newData + (uint32)jmSize, data, (uint32)size);
    entries[entry.chunk] = entry;
    delete[] jmData;
    jmData = newData;
    jmSize = newSize;
    updateMultipartJSON();
    mutex.leave();
    return entry.chunk;
}
 
std::string JSONM::getInfoString() {
    if (!mutex.enter(1000)) return "";
    if (!jmData || !jmSize) {
        mutex.leave();
        return "";
    }
    std::string info = utils::StringFormat("JSON length: %u\nTotal size: %s\nChunks: %u   total: %s\n%s\n",
        mOffset - 1,
        utils::BytifySize((uint32)jmSize).c_str(),
        getCount(),
        utils::BytifySize((uint32)(jmSize - mOffset)).c_str(),
        jmData);
    std::map<uint32, JSONMEntry>::iterator i = entries.begin(), e = entries.end();
    while (i != e) {
        info += i->second.getInfoString();
        i++;
    }
    mutex.leave();
    return info;
}
 
 
 
 
bool JSONM::extractMultipartInfo() {
    // Assume multipart info is in JSON
    // read and create entries from this
    // also assume mutex is already locked
 
    if (!jmData || !jmSize || (jmSize - mOffset < 10))
        return false;
 
    jsmn_parser parser;
    jsmntok_t* tokens = new jsmntok_t[1024];
    int tokenCount = 0;
    //const char* val;
    //int valSize;
    jsmn_init(&parser);
    if ((tokenCount = jsmn_parse(&parser, jmData, strlen(jmData), tokens, 1024)) <= 0) {
        // No JSON found
        delete[] tokens;
        return false;
    }
 
    JSONMEntry entry;
    //int token;
    std::vector<int> jsonArrayIdx;
    std::vector<int>::iterator i, e;
    jsonArrayIdx = GetJSONChildArrayIndexes(tokens, tokenCount, jmData, "_multipart_", 0, JSMN_UNDEFINED);
    if (!jsonArrayIdx.size()) {
        // No info found
        delete[] tokens;
        return false;
    }
    i = jsonArrayIdx.begin();
    e = jsonArrayIdx.end();
 
    uint64 offset = 0;
    while (i != e) {
        if ((entry.name = GetJSONChildValueString(tokens, tokenCount, jmData, "name", *i)).length()) {
            entry.type = GetJSONChildValueString(tokens, tokenCount, jmData, "mimetype", *i);
            entry.size = GetJSONChildValueUint64(tokens, tokenCount, jmData, "bytesize", *i);
            entry.chunk = (uint32)entries.size() + 1;
            entry.offset = offset;
            entries[entry.chunk] = entry;
            offset += entry.size;
        }
        i++;
    }
    return true;
}
 
bool JSONM::updateMultipartJSON() {
    // Assume entries and binary chunks updated
    // create new JSON entry and replace old one, if present
    // also assume mutex is already locked
 
    std::string newJSON;
 
    std::map<uint32, JSONMEntry>::iterator i = entries.begin(), e = entries.end();
    while (i != e) {
        if (newJSON.length())
            newJSON += ",";
        newJSON += i->second.toJSON();
        i++;
    }
 
    jsmn_parser parser;
    jsmntok_t* tokens = new jsmntok_t[1024];
    int tokenCount = 0;
    //const char* val;
    //int valSize;
    jsmn_init(&parser);
    if ((tokenCount = jsmn_parse(&parser, jmData, strlen(jmData), tokens, 1024)) <= 0) {
        // No JSON found
        delete[] tokens;
        return false;
    }
 
    std::string fullJSON = jmData;
    JSONMEntry entry;
    int token = GetJSONToken(tokens, tokenCount, jmData, "_multipart_", 0);
    if (token > 0) {
        uint32 begin = tokens[token].start;
        uint32 end = tokens[token].end;
        //uint32 end = tokens[tokens[tokens[token].parent].parent].end;
        fullJSON = utils::StringFormat("%s[%s]%s",
            fullJSON.substr(0, begin).c_str(),
            newJSON.c_str(),
            fullJSON.substr(end).c_str()
        );
    }
    else {
        // just add at the end
        const char* lastBracket = utils::laststrstr(jmData, "}");
        if (!lastBracket) {
            // Broken JSON found
            delete[] tokens;
            return false;
        }
        fullJSON = utils::StringFormat("%s,\"_multipart_\": [%s]}", fullJSON.substr(0, lastBracket-jmData).c_str(), newJSON.c_str());
    }
    this->setRawJSON(fullJSON.c_str());
    return true;
}
 
DataMessage* JSONM::convertToMessage() {
    DataMessage* msg = new DataMessage();
    uint64 now = GetTimeNow();
 
    msg->setString("POST", this->getJSON().c_str());
    msg->setString("POST_CONTENT_TYPE_", "application/json");
 
    std::map<uint32, JSONMEntry>::iterator i = entries.begin(), e = entries.end();
    while (i != e) {
        msg->setData(i->second.name.c_str(), jmData + mOffset + i->second.offset, (uint32)i->second.size);
        msg->setString((i->second.name + "_CONTENT_TYPE_").c_str(), i->second.type.c_str());
        i++;
    }
    msg->setCreatedTime(now);
    msg->setRecvTime(now);
    return msg;
}
 
 
// ################# Unit Test #################
 
// Encode a WebsocketData frame, then decode it back through the
// processHeader/processContent path and verify the recovered payload matches.
// All in-memory, no sockets. Returns true on success.
static bool TestWebsocketRoundTrip(WebsocketData::DataType type, bool maskData,
    const char* payload, uint64 payloadSize) {
 
    WebsocketData enc;
    if (!enc.setData(type, maskData, payload, payloadSize)) {
        unittest::fail("WebsocketData::setData returned false (type %d, mask %d, size %llu)",
            (int)type, (int)maskData, payloadSize);
        return false;
    }
    if (!enc.isComplete()) {
        unittest::fail("encoded WebsocketData is not COMPLETE");
        return false;
    }
 
    uint64 rawSize = 0;
    const char* raw = enc.getRawData(rawSize);
    if (!raw || !rawSize) {
        unittest::fail("getRawData returned no bytes");
        return false;
    }
    if (rawSize != enc.headerLength + payloadSize) {
        unittest::fail("raw size %llu != header %u + payload %llu",
            rawSize, enc.headerLength, payloadSize);
        return false;
    }
 
    // First byte must carry FIN + opcode; opcode must equal the data type.
    uint8 b0 = *(const unsigned char*)raw;
    if (!(b0 & WebsocketData::BHB0_FIN)) {
        unittest::fail("encoded frame missing FIN bit");
        return false;
    }
    if ((b0 & WebsocketData::BHB0_OPCODE) != (uint8)type) {
        unittest::fail("encoded opcode %u != type %u", (uint8)(b0 & WebsocketData::BHB0_OPCODE), (uint8)type);
        return false;
    }
 
    // Decode: split the raw bytes into header + payload, exactly as the socket
    // reader would, but feeding from the in-memory buffer.
    WebsocketData dec;
    bool isInvalid = false;
    if (!dec.processHeader(raw, rawSize, isInvalid) || isInvalid) {
        unittest::fail("processHeader failed (invalid=%d)", (int)isInvalid);
        return false;
    }
    if (dec.dataType != type) {
        unittest::fail("decoded dataType %d != %d", (int)dec.dataType, (int)type);
        return false;
    }
    if (dec.getPayloadSize() != payloadSize) {
        unittest::fail("decoded payloadSize %llu != %llu", dec.getPayloadSize(), payloadSize);
        return false;
    }
 
    if (payloadSize) {
        if (!dec.processContent(raw + dec.headerLength, payloadSize)) {
            unittest::fail("processContent failed");
            return false;
        }
        uint64 outSize = 0;
        const char* out = dec.getContent(outSize);
        if (!out || (outSize != payloadSize)) {
            unittest::fail("getContent size %llu != %llu", outSize, payloadSize);
            return false;
        }
        if (memcmp(out, payload, (size_t)payloadSize) != 0) {
            unittest::fail("decoded payload bytes differ from original");
            return false;
        }
    }
    return true;
}
 
bool NetworkProtocols_UnitTest() {
 
    // ---- Phase 1: WebsocketData encode/decode round-trips ----
    unittest::progress(5, "websocket frames");
 
    const char* shortText = "hello websocket";
    if (!TestWebsocketRoundTrip(WebsocketData::TEXT, false, shortText, strlen(shortText)))
        return false;
    unittest::detail("websocket short unmasked TEXT round-trip ok (%zu bytes)", strlen(shortText));
 
    // Masked frame: exercises the client->server masking path.
    if (!TestWebsocketRoundTrip(WebsocketData::TEXT, true, shortText, strlen(shortText)))
        return false;
    unittest::detail("websocket short masked TEXT round-trip ok");
 
    if (!TestWebsocketRoundTrip(WebsocketData::BINARY, false, shortText, strlen(shortText)))
        return false;
 
    unittest::progress(25, "websocket extended frames");
 
    // Extended (16-bit length) payload: > 125 bytes forces the 2-byte length field.
    std::string big;
    big.reserve(4096);
    for (uint32 i = 0; i < 4000; i++)
        big.push_back((char)('A' + (i % 26)));
    if (!TestWebsocketRoundTrip(WebsocketData::BINARY, false, big.data(), big.size()))
        return false;
    if (!TestWebsocketRoundTrip(WebsocketData::BINARY, true, big.data(), big.size()))
        return false;
    unittest::detail("websocket extended (16-bit length) masked+unmasked round-trip ok (%zu bytes)", big.size());
 
    // ---- Phase 2: HTTP request build/parse round-trip ----
    unittest::progress(50, "http request");
 
    const char* body = "field1=value1&field2=value2";
    uint32 bodyLen = (uint32)strlen(body);
 
    // Put query parameters in the URI so processHeader -> parseURIParameters
    // populates params; the urlencoded body is carried verbatim and checked
    // byte-for-byte below.
    const char* fullURI = "/path/script.cgi?home=Cosby&favorite=flies";
 
    HTTPRequest reqEnc;
    if (!reqEnc.createRequest(HTTP_POST, "example.com", fullURI,
            body, bodyLen, true, 0)) {
        unittest::fail("HTTPRequest::createRequest returned false");
        return false;
    }
 
    uint32 rawSize = 0;
    const char* raw = reqEnc.getRawContent(rawSize);
    if (!raw || !rawSize) {
        unittest::fail("HTTPRequest::getRawContent returned no bytes");
        return false;
    }
 
    // Parse it back through a fresh request, feeding header then content
    // from the in-memory buffer (header ends at reqEnc.headerLength).
    HTTPRequest reqDec;
    bool isInvalid = false;
    if (!reqDec.processHeader(raw, rawSize, isInvalid) || isInvalid) {
        unittest::fail("HTTPRequest::processHeader failed (invalid=%d)", (int)isInvalid);
        return false;
    }
    if (reqDec.type != HTTP_POST) {
        unittest::fail("parsed method %u != HTTP_POST", reqDec.type);
        return false;
    }
    const char* uri = reqDec.getURI();
    if (!uri || strcmp(uri, fullURI) != 0) {
        unittest::fail("parsed URI '%s' != '%s'", uri ? uri : "(null)", fullURI);
        return false;
    }
    const char* request = reqDec.getRequest();
    if (!request || strcmp(request, "path/script.cgi") != 0) {
        unittest::fail("parsed request '%s' != 'path/script.cgi'", request ? request : "(null)");
        return false;
    }
    if (reqDec.contentLength != bodyLen) {
        unittest::fail("parsed Content-Length %u != %u", reqDec.contentLength, bodyLen);
        return false;
    }
 
    unittest::progress(70, "http content");
 
    // Feed the body (located right after the header in the same buffer).
    if (!reqDec.processContent(raw + reqDec.headerLength, bodyLen)) {
        unittest::fail("HTTPRequest::processContent failed");
        return false;
    }
    uint32 outSize = 0;
    const char* outBody = reqDec.getContent(outSize);
    if (!outBody || (outSize != bodyLen) || (memcmp(outBody, body, bodyLen) != 0)) {
        unittest::fail("parsed body differs from original (size %u != %u)", outSize, bodyLen);
        return false;
    }
    // query-string params (from the URI) should have been parsed.
    const char* p = reqDec.getParameter("home");
    if (!p || strcmp(p, "Cosby") != 0) {
        unittest::fail("parsed URI param home='%s' != 'Cosby'", p ? p : "(null)");
        return false;
    }
    unittest::detail("http POST build/parse round-trip ok (body %u bytes)", bodyLen);
 
    // ---- Phase 3: parse throughput metric ----
    unittest::progress(85, "throughput");
 
    const uint32 iters = 2000;
    uint64 t0 = GetTimeNow();
    for (uint32 i = 0; i < iters; i++) {
        HTTPRequest r;
        bool inv = false;
        if (!r.processHeader(raw, reqEnc.headerLength, inv) || inv) {
            unittest::fail("throughput loop processHeader failed at iter %u", i);
            return false;
        }
    }
    double us = (double)(GetTimeNow() - t0);
    if (us > 0.0)
        unittest::metric("http_header_parse_throughput", (double)iters / us * 1e6, "ops/s", true);
 
    unittest::progress(100, "done");
    return true;
}
 
void Register_NetworkProtocols_Tests() {
    UnitTestRunner::instance().registerTest(
        "networkprotocols", NetworkProtocols_UnitTest,
        "HTTP request and WebSocket frame build/parse round-trips", "network");
}
 
}