// SPDX-License-Identifier: LGPL-2.1-or-later /*************************************************************************** * Copyright (c) 2019 Jean-Marie Verdun jmverdun3@gmail.com * * * * This file is part of the FreeCAD CAx development system. * * * * This library is free software; you can redistribute it and/or * * modify it under the terms of the GNU Library General Public * * License as published by the Free Software Foundation; either * * version 2 of the License, or (at your option) any later version. * * * * This library is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU Library General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this library; see the file COPYING.LIB. If not, * * write to the Free Software Foundation, Inc., 59 Temple Place, * * Suite 330, Boston, MA 02111-1307, USA * * * ***************************************************************************/ #include #include #if defined(FC_OS_WIN32) # include # include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "AppCloud.h" using namespace App; using namespace std; using namespace boost::placeholders; XERCES_CPP_NAMESPACE_USE /* Python entry */ PyMOD_INIT_FUNC(Cloud) { PyObject* mod = Cloud::initModule(); Base::Console().log("Loading Cloud module... done\n"); PyMOD_Return(mod); } Py::Object Cloud::Module::sCloudURL(const Py::Tuple& args) { char* URL; if (!PyArg_ParseTuple(args.ptr(), "et", "utf-8", &URL)) { throw Py::Exception(); } std::string strURL = URL; PyMem_Free(URL); if (this->URL.getStrValue() != strURL) { this->URL.setValue(strURL); } return Py::None(); } Py::Object Cloud::Module::sCloudTokenAuth(const Py::Tuple& args) { char* TokenAuth; if (!PyArg_ParseTuple(args.ptr(), "et", "utf-8", &TokenAuth)) { throw Py::Exception(); } std::string strTokenAuth = TokenAuth; PyMem_Free(TokenAuth); if (this->TokenAuth.getStrValue() != strTokenAuth) { this->TokenAuth.setValue(strTokenAuth); } return Py::None(); } Py::Object Cloud::Module::sCloudTokenSecret(const Py::Tuple& args) { char* TokenSecret; if (!PyArg_ParseTuple(args.ptr(), "et", "utf-8", &TokenSecret)) { throw Py::Exception(); } std::string strTokenSecret = TokenSecret; PyMem_Free(TokenSecret); if (this->TokenSecret.getStrValue() != strTokenSecret) { this->TokenSecret.setValue(strTokenSecret); } return Py::None(); } Py::Object Cloud::Module::sCloudTCPPort(const Py::Tuple& args) { char* TCPPort; if (!PyArg_ParseTuple(args.ptr(), "et", "utf-8", &TCPPort)) { throw Py::Exception(); } std::string strTCPPort = TCPPort; PyMem_Free(TCPPort); if (this->TCPPort.getStrValue() != strTCPPort) { this->TCPPort.setValue(strTCPPort); } return Py::None(); } Py::Object Cloud::Module::sCloudSave(const Py::Tuple& args) { char* pDoc; if (!PyArg_ParseTuple(args.ptr(), "et", "utf-8", &pDoc)) { throw Py::Exception(); } std::string strpDoc = pDoc; PyMem_Free(pDoc); cloudSave(strpDoc.c_str()); return Py::None(); } Py::Object Cloud::Module::sCloudRestore(const Py::Tuple& args) { char* pDoc; if (!PyArg_ParseTuple(args.ptr(), "et", "utf-8", &pDoc)) { throw Py::Exception(); } std::string strpDoc = pDoc; PyMem_Free(pDoc); cloudRestore(strpDoc.c_str()); return Py::None(); } Py::Object Cloud::Module::sCloudProtocolVersion(const Py::Tuple& args) { char* ProtocolVersion; if (!PyArg_ParseTuple(args.ptr(), "et", "utf-8", &ProtocolVersion)) { throw Py::Exception(); } std::string strProtocolVersion = ProtocolVersion; PyMem_Free(ProtocolVersion); if (this->ProtocolVersion.getStrValue() != strProtocolVersion) { this->ProtocolVersion.setValue(strProtocolVersion); } return Py::None(); } Py::Object Cloud::Module::sCloudRegion(const Py::Tuple& args) { char* Region; if (!PyArg_ParseTuple(args.ptr(), "et", "utf-8", &Region)) { throw Py::Exception(); } std::string strRegion = Region; PyMem_Free(Region); if (this->Region.getStrValue() != strRegion) { this->Region.setValue(strRegion); } return Py::None(); } struct data_buffer { const char* ptr; size_t remaining_size; }; static size_t read_callback(void* ptr, size_t size, size_t nmemb, void* stream) { struct data_buffer* local_ptr = (struct data_buffer*)stream; size_t data_to_transfer = size * nmemb; if (local_ptr->remaining_size) { // copy as much as possible from the source to the destination size_t copy_this_much = local_ptr->remaining_size; if (copy_this_much > data_to_transfer) { copy_this_much = data_to_transfer; } memcpy(ptr, local_ptr->ptr, copy_this_much); local_ptr->ptr += copy_this_much; local_ptr->remaining_size -= copy_this_much; return copy_this_much; } return 0; } void Cloud::CloudWriter::checkXML(DOMNode* node) { if (node) { switch (node->getNodeType()) { case DOMNode::ELEMENT_NODE: checkElement(static_cast(node)); break; case DOMNode::TEXT_NODE: checkText(static_cast(node)); break; default: break; } DOMNode* child = node->getFirstChild(); while (child) { DOMNode* next = child->getNextSibling(); checkXML(child); child = next; } } } void Cloud::CloudWriter::checkElement(DOMElement* element) { char* name = XMLString::transcode(element->getTagName()); if (strcmp(name, "Code") == 0) { print = 1; } XMLString::release(&name); } void Cloud::CloudWriter::checkText(DOMText* text) { XMLCh* buffer = new XMLCh[XMLString::stringLen(text->getData()) + 1]; XMLString::copyString(buffer, text->getData()); XMLString::trim(buffer); char* content = XMLString::transcode(buffer); delete[] buffer; if (print) { strcpy(errorCode, content); } print = 0; XMLString::release(&content); } void Cloud::CloudWriter::createBucket() { struct Cloud::AmzData* RequestData; struct Cloud::AmzDatav4* RequestDatav4; CURL* curl; CURLcode res; struct data_buffer curl_buffer; char path[1024]; sprintf(path, "/%s/", this->Bucket); std::string strURL(this->URL); eraseSubStr(strURL, "http://"); eraseSubStr(strURL, "https://"); if (this->ProtocolVersion == "2") { RequestData = Cloud::ComputeDigestAmzS3v2("PUT", "application/xml", path, this->TokenSecret, nullptr, 0); } else { RequestDatav4 = Cloud::ComputeDigestAmzS3v4( "PUT", strURL.c_str(), "application/xml", path, this->TokenSecret, nullptr, 0, nullptr, this->Region ); } // Let's build the Header and call to curl curl_global_init(CURL_GLOBAL_ALL); curl = curl_easy_init(); #ifdef ALLOW_SELF_SIGNED_CERTIFICATE curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); #endif if (curl) { struct curl_slist* chunk = nullptr; char URL[256]; // Let's build our own header std::string strURL(this->URL); eraseSubStr(strURL, "http://"); eraseSubStr(strURL, "https://"); if (this->ProtocolVersion == "2") { chunk = Cloud::BuildHeaderAmzS3v2(strURL.c_str(), this->TCPPort, this->TokenAuth, RequestData); delete RequestData; } else { chunk = Cloud::BuildHeaderAmzS3v4(strURL.c_str(), this->TokenAuth, RequestDatav4); delete RequestDatav4; } curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk); // Lets build the URL for our Curl call sprintf(URL, "%s:%s/%s/", this->URL, this->TCPPort, this->Bucket); curl_easy_setopt(curl, CURLOPT_URL, URL); curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L); curl_easy_setopt(curl, CURLOPT_PUT, 1L); // curl read a file not a memory buffer (it shall be able to do it) curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_callback); curl_buffer.ptr = nullptr; curl_buffer.remaining_size = (size_t)0; curl_easy_setopt(curl, CURLOPT_READDATA, &curl_buffer); curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)0); res = curl_easy_perform(curl); if (res != CURLE_OK) { fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res)); } curl_easy_cleanup(curl); } } struct Cloud::AmzDatav4* Cloud::ComputeDigestAmzS3v4( char* operation, const char* server, char* data_type, const char* target, const char* Secret, const char* ptr, long size, char* parameters, std::string Region ) { struct AmzDatav4* returnData; returnData = new Cloud::AmzDatav4; struct tm* tm; char* canonical_request; char* canonicalRequestHash; char* stringToSign; strcpy(returnData->ContentType, data_type); #if defined(FC_OS_WIN32) _putenv("TZ=GMT"); time_t rawtime; time(&rawtime); tm = localtime(&rawtime); #else struct timeval tv; setenv("TZ", "GMT", 1); gettimeofday(&tv, nullptr); tm = localtime(&tv.tv_sec); #endif strftime(returnData->dateFormattedD, 256, "%Y%m%d", tm); strftime(returnData->dateFormattedS, 256, "%Y%m%dT%H%M%SZ", tm); returnData->MD5 = nullptr; // We must evaluate the canonical request canonical_request = (char*)malloc(4096 * (sizeof(char*))); strcpy(canonical_request, operation); strcat(canonical_request, "\n"); strcat(canonical_request, target); strcat(canonical_request, "\n"); if (parameters == nullptr) { strcat(canonical_request, "\n"); } else { strcat(canonical_request, parameters); strcat(canonical_request, "\n"); } strcat(canonical_request, "host:"); strcat(canonical_request, server); strcat(canonical_request, "\n"); strcat(canonical_request, "x-amz-date:"); strcat(canonical_request, returnData->dateFormattedS); strcat(canonical_request, "\n\n"); strcat(canonical_request, "host;x-amz-date\n"); // We must add there the file SHA256 Hash returnData->SHA256Sum = nullptr; if (strcmp(operation, "PUT") == 0) { if (ptr != nullptr) { returnData->SHA256Sum = Cloud::SHA256Sum(ptr, size); strcat(canonical_request, returnData->SHA256Sum); } else { strcat(canonical_request, "UNSIGNED-PAYLOAD"); } } else { strcat(canonical_request, "UNSIGNED-PAYLOAD"); } canonicalRequestHash = Cloud::SHA256Sum(canonical_request, strlen(canonical_request)); // returnData->digest = string(digest); // We need now to sign a string which contain the digest // The format is as follow // ${authType} // ${dateValueL} // ${dateValueS}/${Region}/${service}/aws4_request // ${canonicalRequestHash}" stringToSign = (char*)malloc(4096 * sizeof(char)); strcat(stringToSign, "AWS4-HMAC-SHA256"); strcat(stringToSign, "\n"); strcat(stringToSign, returnData->dateFormattedS); strcat(stringToSign, "\n"); strcat(stringToSign, returnData->dateFormattedD); strcat(stringToSign, "/us/s3/aws4_request"); strcat(stringToSign, "\n"); strcat(stringToSign, canonicalRequestHash); strcat(stringToSign, "\0"); // We must now compute the signature // Everything starts with the secret key and an SHA256 HMAC encryption char kSecret[256]; unsigned char *kDate, *kRegion, *kService, *kSigned, *kSigning; strcpy(kSecret, "AWS4"); strcat(kSecret, Secret); unsigned int HMACLength; std::string temporary; kDate = HMAC( EVP_sha256(), kSecret, strlen(kSecret), (const unsigned char*)returnData->dateFormattedD, strlen(returnData->dateFormattedD), nullptr, &HMACLength ); temporary = getHexValue(kDate, HMACLength); temporary = getHexValue(kDate, HMACLength); // We can now compute the remaining parts kRegion = HMAC( EVP_sha256(), kDate, HMACLength, (const unsigned char*)Region.c_str(), strlen(Region.c_str()), nullptr, &HMACLength ); temporary = getHexValue(kRegion, HMACLength); kService = HMAC( EVP_sha256(), kRegion, HMACLength, (const unsigned char*)"s3", strlen("s3"), nullptr, &HMACLength ); temporary = getHexValue(kService, HMACLength); kSigning = HMAC( EVP_sha256(), kService, HMACLength, (const unsigned char*)"aws4_request", strlen("aws4_request"), nullptr, &HMACLength ); temporary = getHexValue(kService, HMACLength); kSigned = HMAC( EVP_sha256(), kSigning, HMACLength, (const unsigned char*)stringToSign, strlen(stringToSign), nullptr, &HMACLength ); temporary = getHexValue(kSigned, HMACLength); returnData->digest = string(temporary); returnData->Region = Region; free(canonical_request); return (returnData); } std::string Cloud::getHexValue(unsigned char* input, unsigned int HMACLength) { char* Hex; unsigned char* ptr = input; std::string resultReadable; Hex = (char*)malloc(HMACLength * 2 * sizeof(char) + 1); for (unsigned int i = 0; i < HMACLength; i++) { sprintf(Hex, "%02x", *ptr); ptr++; Hex[2] = '\0'; resultReadable += Hex; } return resultReadable; } struct Cloud::AmzData* Cloud::ComputeDigestAmzS3v2( char* operation, char* data_type, const char* target, const char* Secret, const char* ptr, long size ) { struct AmzData* returnData; // struct timeval tv; struct tm* tm; char date_formatted[256]; char StringToSign[1024]; unsigned char* digest; unsigned int HMACLength; // Amazon S3 and Swift require the timezone to be define to GMT. // As to simplify the conversion this is performed through the TZ // environment variable and a call to localtime as to convert output of gettimeofday returnData = new Cloud::AmzData; strcpy(returnData->ContentType, data_type); #if defined(FC_OS_WIN32) _putenv("TZ=GMT"); time_t rawtime; time(&rawtime); tm = localtime(&rawtime); #else struct timeval tv; setenv("TZ", "GMT", 1); gettimeofday(&tv, nullptr); tm = localtime(&tv.tv_sec); #endif strftime(date_formatted, 256, "%a, %d %b %Y %T %z", tm); returnData->MD5 = nullptr; if (strcmp(operation, "PUT") == 0) { if (ptr != nullptr) { returnData->MD5 = Cloud::MD5Sum(ptr, size); sprintf( StringToSign, "%s\n%s\n%s\n%s\n%s", operation, returnData->MD5, data_type, date_formatted, target ); } else { sprintf(StringToSign, "%s\n\n%s\n%s\n%s", operation, data_type, date_formatted, target); } } else { sprintf(StringToSign, "%s\n\n%s\n%s\n%s", operation, data_type, date_formatted, target); } // We have to use HMAC encoding and SHA1 digest = HMAC( EVP_sha1(), Secret, strlen(Secret), (const unsigned char*)&StringToSign, strlen(StringToSign), nullptr, &HMACLength ); returnData->digest = Base::base64_encode(digest, HMACLength); strcpy(returnData->dateFormatted, date_formatted); return returnData; } char* Cloud::SHA256Sum(const char* ptr, long size) { char* output; std::string local; std::string resultReadable; unsigned char result[SHA256_DIGEST_LENGTH]; output = (char*)malloc(2 * SHA256_DIGEST_LENGTH * sizeof(char) + 1); SHA256((unsigned char*)ptr, size, result); strcpy(output, getHexValue(result, SHA256_DIGEST_LENGTH).c_str()); return (output); } char* Cloud::MD5Sum(const char* ptr, long size) { char* output; std::string local; unsigned char result[MD5_DIGEST_LENGTH]; output = (char*)malloc(2 * MD5_DIGEST_LENGTH * sizeof(char) + 1); MD5((unsigned char*)ptr, size, result); local = Base::base64_encode(result, MD5_DIGEST_LENGTH); strcpy(output, local.c_str()); return (output); } struct curl_slist* Cloud::BuildHeaderAmzS3v4( const char* URL, const char* PublicKey, struct Cloud::AmzDatav4* Data ) { char header_data[1024]; struct curl_slist* chunk = nullptr; // Build the Host: entry sprintf(header_data, "Host: %s", URL); chunk = curl_slist_append(chunk, header_data); // Build the Date entry sprintf(header_data, "X-Amz-Date: %s", Data->dateFormattedS); chunk = curl_slist_append(chunk, header_data); // Build the Content-Type entry sprintf(header_data, "Content-Type:%s", Data->ContentType); chunk = curl_slist_append(chunk, header_data); // If ptr is not null we must compute the MD5-Sum as to validate later the ETag // and add the MD5-Content: entry to the header if (Data->MD5 != nullptr) { sprintf(header_data, "Content-MD5: %s", Data->MD5); chunk = curl_slist_append(chunk, header_data); // We don't need it anymore we can free it free((void*)Data->MD5); } if (Data->SHA256Sum != nullptr) { sprintf(header_data, "x-amz-content-sha256: %s", Data->SHA256Sum); chunk = curl_slist_append(chunk, header_data); // We don't need it anymore we can free it free((void*)Data->SHA256Sum); } else { chunk = curl_slist_append(chunk, "x-amz-content-sha256: UNSIGNED-PAYLOAD"); } // build the Auth entry sprintf( header_data, "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/%s/aws4_request, " "SignedHeaders=%s, Signature=%s", PublicKey, Data->dateFormattedD, "us", "s3", "host;x-amz-date", Data->digest.c_str() ); chunk = curl_slist_append(chunk, header_data); return chunk; } struct curl_slist* Cloud::BuildHeaderAmzS3v2( const char* URL, const char* TCPPort, const char* PublicKey, struct Cloud::AmzData* Data ) { char header_data[1024]; struct curl_slist* chunk = nullptr; // Build the Host: entry sprintf(header_data, "Host: %s:%s", URL, TCPPort); chunk = curl_slist_append(chunk, header_data); // Build the Date entry sprintf(header_data, "Date: %s", Data->dateFormatted); chunk = curl_slist_append(chunk, header_data); // Build the Content-Type entry sprintf(header_data, "Content-Type:%s", Data->ContentType); chunk = curl_slist_append(chunk, header_data); // If ptr is not null we must compute the MD5-Sum as to validate later the ETag // and add the MD5-Content: entry to the header if (Data->MD5 != nullptr) { sprintf(header_data, "Content-MD5: %s", Data->MD5); chunk = curl_slist_append(chunk, header_data); // We don't need it anymore we can free it free((void*)Data->MD5); } // build the Auth entry sprintf(header_data, "Authorization: AWS %s:%s", PublicKey, Data->digest.c_str()); chunk = curl_slist_append(chunk, header_data); return chunk; } Cloud::CloudWriter::CloudWriter( const char* URL, const char* TokenAuth, const char* TokenSecret, const char* TCPPort, const char* Bucket, std::string ProtocolVersion, std::string Region ) { struct Cloud::AmzData* RequestData; struct Cloud::AmzDatav4* RequestDatav4; CURL* curl; CURLcode res; std::string s; this->URL = URL; this->TokenAuth = TokenAuth; this->TokenSecret = TokenSecret; this->TCPPort = TCPPort; this->Bucket = Bucket; if (!ProtocolVersion.empty()) { this->ProtocolVersion = ProtocolVersion; } else { this->ProtocolVersion = "2"; } this->Region = Region; this->FileName = ""; char path[1024]; sprintf(path, "/%s/", this->Bucket); std::string strURL(this->URL); eraseSubStr(strURL, "http://"); eraseSubStr(strURL, "https://"); if (this->ProtocolVersion == "2") { RequestData = Cloud::ComputeDigestAmzS3v2("GET", "application/xml", path, this->TokenSecret, nullptr, 0); } else { RequestDatav4 = Cloud::ComputeDigestAmzS3v4( "GET", strURL.c_str(), "application/xml", path, this->TokenSecret, nullptr, 0, nullptr, this->Region ); } // Let's build the Header and call to curl curl_global_init(CURL_GLOBAL_ALL); curl = curl_easy_init(); #ifdef ALLOW_SELF_SIGNED_CERTIFICATE curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); #endif if (curl) { // Let's build our own header struct curl_slist* chunk = nullptr; char URL[256]; std::string strURL(this->URL); eraseSubStr(strURL, "http://"); eraseSubStr(strURL, "https://"); if (this->ProtocolVersion == "2") { chunk = Cloud::BuildHeaderAmzS3v2(strURL.c_str(), this->TCPPort, this->TokenAuth, RequestData); delete RequestData; } else { chunk = Cloud::BuildHeaderAmzS3v4(strURL.c_str(), this->TokenAuth, RequestDatav4); delete RequestDatav4; } curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk); // Lets build the URL for our Curl call sprintf(URL, "%s:%s/%s/", this->URL, this->TCPPort, this->Bucket); curl_easy_setopt(curl, CURLOPT_URL, URL); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, CurlWrite_CallbackFunc_StdString); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &s); // curl read a file not a memory buffer (it shall be able to do it) res = curl_easy_perform(curl); if (res != CURLE_OK) { fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res)); } curl_easy_cleanup(curl); createBucket(); // Lets dump temporarily for debug purposes of s3v4 implementation std::stringstream input(s); try { XMLPlatformUtils::Initialize(); } catch (const XMLException& toCatch) { char* message = XMLString::transcode(toCatch.getMessage()); cout << "Error during initialization! :\n" << message << "\n"; XMLString::release(&message); return; } XercesDOMParser* parser = new XercesDOMParser(); parser->setValidationScheme(XercesDOMParser::Val_Always); parser->setDoNamespaces(true); xercesc::MemBufInputSource myxml_buf( (const XMLByte* const)s.c_str(), s.size(), "myxml (in memory)" ); parser->parse(myxml_buf); auto* dom = parser->getDocument(); // Is there an Error entry into the document ? // if yes, then we must create the Bucket checkXML(dom); if (strcmp(errorCode, "NoSuchBucket") == 0) { // we must create the Bucket using a PUT request createBucket(); } } } Cloud::CloudWriter::~CloudWriter() {} size_t Cloud::CurlWrite_CallbackFunc_StdString(void* contents, size_t size, size_t nmemb, std::string* s) { size_t newLength = size * nmemb; try { s->append((char*)contents, newLength); } catch (std::bad_alloc&) { // handle memory problem return 0; } return newLength; } void Cloud::CloudReader::checkElement(DOMElement* element) { char* name = XMLString::transcode(element->getTagName()); if (strcmp(name, "Key") == 0) { file = 1; } if (strcmp(name, "NextContinuationToken") == 0) { continuation = 1; } if (strcmp(name, "IsTruncated") == 0) { truncated = 1; } XMLString::release(&name); } void Cloud::CloudReader::checkText(DOMText* text) { XMLCh* buffer = new XMLCh[XMLString::stringLen(text->getData()) + 1]; XMLString::copyString(buffer, text->getData()); XMLString::trim(buffer); struct Cloud::CloudReader::FileEntry* new_entry; char* content = XMLString::transcode(buffer); delete[] buffer; if (file) { new_entry = new Cloud::CloudReader::FileEntry; strcpy(new_entry->FileName, content); Cloud::CloudReader::FileList.push_back(new_entry); } file = 0; if (continuation == 1) { strcpy(Cloud::CloudReader::NextFileName, content); continuation = 0; } if (truncated == 1) { if (strncmp(content, "true", 4) != 0) { truncated = 0; } else { truncated = 2; } } XMLString::release(&content); } void Cloud::CloudReader::addFile(struct Cloud::CloudReader::FileEntry* new_entry) { Cloud::CloudReader::FileList.push_back(new_entry); } void Cloud::CloudReader::checkXML(DOMNode* node) { if (node) { switch (node->getNodeType()) { case DOMNode::ELEMENT_NODE: checkElement(static_cast(node)); break; case DOMNode::TEXT_NODE: checkText(static_cast(node)); break; default: break; } DOMNode* child = node->getFirstChild(); while (child) { DOMNode* next = child->getNextSibling(); checkXML(child); child = next; } } } Cloud::CloudReader::~CloudReader() {} Cloud::CloudReader::CloudReader( const char* URL, const char* TokenAuth, const char* TokenSecret, const char* TCPPort, const char* Bucket, std::string ProtocolVersion, std::string Region ) { struct Cloud::AmzData* RequestData; struct Cloud::AmzDatav4* RequestDatav4; CURL* curl; CURLcode res; bool GetBucketContentList = true; struct curl_slist* chunk = nullptr; char parameters[1024]; this->URL = URL; this->TokenAuth = TokenAuth; this->TokenSecret = TokenSecret; this->TCPPort = TCPPort; this->Bucket = Bucket; if (!ProtocolVersion.empty()) { this->ProtocolVersion = ProtocolVersion; } else { this->ProtocolVersion = "2"; } this->Region = Region; char path[1024]; sprintf(path, "/%s/", this->Bucket); Cloud::CloudReader::NextFileName = (char*)malloc(sizeof(char) * 1024); for (int i = 0; i < 1024; i++) { NextFileName[i] = '\0'; } // Let's build the Header and call to curl curl_global_init(CURL_GLOBAL_ALL); while (GetBucketContentList) { std::string s; curl = curl_easy_init(); #ifdef ALLOW_SELF_SIGNED_CERTIFICATE curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); #endif if (curl) { // Let's build our own header char URL[256]; std::string strURL(this->URL); eraseSubStr(strURL, "http://"); eraseSubStr(strURL, "https://"); if (strlen(NextFileName) == 0) { sprintf(parameters, "list-type=2"); } else { sprintf(parameters, "list-type=2&continuation-token=%s", NextFileName); } sprintf(URL, "%s:%s/%s/?%s", this->URL, this->TCPPort, this->Bucket, parameters); curl_easy_setopt(curl, CURLOPT_URL, URL); if (this->ProtocolVersion == "2") { RequestData = Cloud::ComputeDigestAmzS3v2( "GET", "application/xml", path, this->TokenSecret, nullptr, 0 ); chunk = Cloud::BuildHeaderAmzS3v2( strURL.c_str(), this->TCPPort, this->TokenAuth, RequestData ); delete RequestData; } else { RequestDatav4 = Cloud::ComputeDigestAmzS3v4( "GET", strURL.c_str(), "application/xml", path, this->TokenSecret, nullptr, 0, (char*)¶meters[0], this->Region ); chunk = Cloud::BuildHeaderAmzS3v4(strURL.c_str(), this->TokenAuth, RequestDatav4); delete RequestDatav4; } curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, CurlWrite_CallbackFunc_StdString); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &s); // curl read a file not a memory buffer (it shall be able to do it) res = curl_easy_perform(curl); for (int i = 0; i < 1024; i++) { NextFileName[i] = '\0'; } if (res != CURLE_OK) { fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res)); } curl_easy_cleanup(curl); std::stringstream input(s); try { XMLPlatformUtils::Initialize(); } catch (const XMLException& toCatch) { char* message = XMLString::transcode(toCatch.getMessage()); cout << "Error during initialization! :\n" << message << "\n"; XMLString::release(&message); return; } XercesDOMParser* parser = new XercesDOMParser(); parser->setValidationScheme(XercesDOMParser::Val_Always); parser->setDoNamespaces(true); xercesc::MemBufInputSource myxml_buf( (const XMLByte* const)s.c_str(), s.size(), "myxml (in memory)" ); parser->parse(myxml_buf); auto* dom = parser->getDocument(); checkXML(dom); } if (truncated == 0) { GetBucketContentList = false; } else { truncated = 0; continuation = 0; file = 0; } } } void Cloud::CloudReader::DownloadFile(Cloud::CloudReader::FileEntry* entry) { struct Cloud::AmzData* RequestData; struct Cloud::AmzDatav4* RequestDatav4; CURL* curl; CURLcode res; std::string s; // We must get the directory content char path[1024]; sprintf(path, "/%s/%s", this->Bucket, entry->FileName); std::string strURL(this->URL); eraseSubStr(strURL, "http://"); eraseSubStr(strURL, "https://"); if (this->ProtocolVersion == "2") { RequestData = Cloud::ComputeDigestAmzS3v2( "GET", "application/octet-stream", path, this->TokenSecret, nullptr, 0 ); } else { RequestDatav4 = Cloud::ComputeDigestAmzS3v4( "GET", strURL.c_str(), "application/octet-stream", path, this->TokenSecret, nullptr, 0, nullptr, this->Region ); } // Let's build the Header and call to curl curl_global_init(CURL_GLOBAL_ALL); curl = curl_easy_init(); #ifdef ALLOW_SELF_SIGNED_CERTIFICATE curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); #endif if (curl) { struct curl_slist* chunk = nullptr; char URL[256]; // Let's build our own header std::string strURL(this->URL); eraseSubStr(strURL, "http://"); eraseSubStr(strURL, "https://"); if (this->ProtocolVersion == "2") { chunk = Cloud::BuildHeaderAmzS3v2(strURL.c_str(), this->TCPPort, this->TokenAuth, RequestData); delete RequestData; } else { chunk = Cloud::BuildHeaderAmzS3v4(strURL.c_str(), this->TokenAuth, RequestDatav4); delete RequestDatav4; } curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk); sprintf(URL, "%s:%s/%s/%s", this->URL, this->TCPPort, this->Bucket, entry->FileName); curl_easy_setopt(curl, CURLOPT_URL, URL); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, CurlWrite_CallbackFunc_StdString); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &s); // curl read a file not a memory buffer (it shall be able to do it) res = curl_easy_perform(curl); if (res != CURLE_OK) { fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res)); } curl_easy_cleanup(curl); entry->FileStream << s; } } struct Cloud::CloudReader::FileEntry* Cloud::CloudReader::GetEntry(std::string FileName) { struct Cloud::CloudReader::FileEntry* current_entry = nullptr; list::const_iterator it1; for (it1 = FileList.begin(); it1 != FileList.end(); ++it1) { if (strcmp(FileName.c_str(), (*it1)->FileName) == 0) { current_entry = (*it1); break; } } if (current_entry != nullptr) { (*it1)->touch = 1; DownloadFile(*it1); } return (current_entry); } int Cloud::CloudReader::isTouched(std::string FileName) { list::const_iterator it1; for (it1 = FileList.begin(); it1 != FileList.end(); ++it1) { if (strcmp(FileName.c_str(), (*it1)->FileName) == 0) { if ((*it1)->touch) { return (1); } else { return (0); } } } return (0); } void Cloud::eraseSubStr(std::string& Str, const std::string& toErase) { size_t pos = Str.find(toErase); if (pos != std::string::npos) { Str.erase(pos, toErase.length()); } } void Cloud::CloudWriter::putNextEntry(const char* file) { this->FileName = file; this->FileStream.str(""); this->FileStream << std::fixed; this->FileStream.precision(std::numeric_limits::digits10 + 1); this->FileStream.setf(ios::fixed, ios::floatfield); this->FileStream.imbue(std::locale::classic()); } bool Cloud::CloudWriter::shouldWrite(const std::string&, const Base::Persistence*) const { return true; } void Cloud::CloudWriter::pushCloud(const char* FileName, const char* data, long size) { struct Cloud::AmzData* RequestData; struct Cloud::AmzDatav4* RequestDatav4; CURL* curl; CURLcode res; struct data_buffer curl_buffer; char path[1024]; sprintf(path, "/%s/%s", this->Bucket, FileName); std::string strURL(this->URL); eraseSubStr(strURL, "http://"); eraseSubStr(strURL, "https://"); if (this->ProtocolVersion == "2") { RequestData = Cloud::ComputeDigestAmzS3v2( "PUT", "application/octet-stream", path, this->TokenSecret, data, size ); } else { RequestDatav4 = Cloud::ComputeDigestAmzS3v4( "PUT", strURL.c_str(), "application/octet-stream", path, this->TokenSecret, data, size, nullptr, this->Region ); } // Let's build the Header and call to curl curl_global_init(CURL_GLOBAL_ALL); curl = curl_easy_init(); #ifdef ALLOW_SELF_SIGNED_CERTIFICATE curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); #endif if (curl) { struct curl_slist* chunk = nullptr; char URL[256]; // Let's build our own header std::string strURL(this->URL); eraseSubStr(strURL, "http://"); eraseSubStr(strURL, "https://"); if (this->ProtocolVersion == "2") { chunk = Cloud::BuildHeaderAmzS3v2(strURL.c_str(), this->TCPPort, this->TokenAuth, RequestData); delete RequestData; } else { chunk = Cloud::BuildHeaderAmzS3v4(strURL.c_str(), this->TokenAuth, RequestDatav4); delete RequestDatav4; } curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk); // Lets build the URL for our Curl call sprintf(URL, "%s:%s/%s/%s", this->URL, this->TCPPort, this->Bucket, FileName); curl_easy_setopt(curl, CURLOPT_URL, URL); curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L); curl_easy_setopt(curl, CURLOPT_PUT, 1L); // curl read a file not a memory buffer (it shall be able to do it) curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_callback); curl_buffer.ptr = data; curl_buffer.remaining_size = (size_t)size; curl_easy_setopt(curl, CURLOPT_READDATA, &curl_buffer); curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)size); res = curl_easy_perform(curl); if (res != CURLE_OK) { fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res)); } curl_easy_cleanup(curl); } } void Cloud::CloudWriter::writeFiles(void) { // use a while loop because it is possible that while // processing the files, new ones can be added std::string tmp = ""; char* cstr; size_t index = 0; if (strlen(this->FileName.c_str()) > 1) { // We must push the current buffer const std::string tmp = this->FileStream.str(); const char* cstr = tmp.data(); pushCloud((const char*)this->FileName.c_str(), cstr, tmp.size()); } while (index < FileList.size()) { FileEntry entry = FileList.begin()[index]; if (shouldWrite(entry.FileName, entry.Object)) { this->FileStream.str(""); this->FileStream.precision(std::numeric_limits::digits10 + 1); this->FileStream.setf(ios::fixed, ios::floatfield); this->FileStream.imbue(std::locale::classic()); entry.Object->SaveDocFile(*this); tmp = this->FileStream.str(); cstr = (char*)tmp.data(); pushCloud((const char*)entry.FileName.c_str(), (const char*)cstr, tmp.size()); this->FileStream.str(""); } index++; } } bool Cloud::Module::cloudSave(const char* BucketName) { Document* doc = GetApplication().getActiveDocument(); auto hGrp = App::GetApplication().GetParameterGroupByPath( "User parameter:BaseApp/Preferences/Document" ); // Save the name of the tip object in order to handle in Restore() if (doc->Tip.getValue()) { doc->TipName.setValue(doc->Tip.getValue()->getNameInDocument()); } std::string LastModifiedDateString = Base::Tools::currentDateTimeString(); doc->LastModifiedDate.setValue(LastModifiedDateString.c_str()); // set author if needed bool saveAuthor = App::GetApplication() .GetParameterGroupByPath("User parameter:BaseApp/Preferences/Document") ->GetBool("prefSetAuthorOnSave", false); if (saveAuthor) { std::string Author = App::GetApplication() .GetParameterGroupByPath("User parameter:BaseApp/Preferences/Document") ->GetASCII("prefAuthor", ""); doc->LastModifiedBy.setValue(Author.c_str()); } if (strcmp(BucketName, doc->Label.getValue()) != 0) { doc->Label.setValue(BucketName); } Cloud::CloudWriter mywriter( (const char*)this->URL.getStrValue().c_str(), (const char*)this->TokenAuth.getStrValue().c_str(), (const char*)this->TokenSecret.getStrValue().c_str(), (const char*)this->TCPPort.getStrValue().c_str(), BucketName, (const char*)this->ProtocolVersion.getStrValue().c_str(), this->Region.getStrValue() ); mywriter.putNextEntry("Document.xml"); if (hGrp->GetBool("SaveBinaryBrep", false)) { mywriter.setMode("BinaryBrep"); } mywriter.Stream() << "" << endl << "" << endl; doc->Save(mywriter); // Special handling for Gui document. doc->signalSaveDocument(mywriter); // write additional files mywriter.writeFiles(); return (true); } void readFiles(Cloud::CloudReader reader, Base::XMLReader* xmlreader) { // It's possible that not all objects inside the document could be created, e.g. if a module // is missing that would know these object types. So, there may be data files inside the Cloud // file that cannot be read. We simply ignore these files. // On the other hand, however, it could happen that a file should be read that is not part of // the zip file. This happens e.g. if a document is written without GUI up but is read with GUI // up. In this case the associated GUI document asks for its file which is not part of the // file, then. // In either case it's guaranteed that the order of the files is kept. std::vector::const_iterator it = xmlreader->FileList.begin(); while (it != xmlreader->FileList.end()) { if (reader.isTouched(it->FileName.c_str()) == 0) { Base::Reader localreader( reader.GetEntry(it->FileName.c_str())->FileStream, it->FileName, xmlreader->FileVersion ); // for debugging only purpose if (false) { std::stringstream ss; ss << localreader.getStream().rdbuf(); auto aString = ss.str(); aString = ""; } it->Object->RestoreDocFile(localreader); if (localreader.getLocalReader() != nullptr) { readFiles(reader, localreader.getLocalReader().get()); } } it++; } } void Cloud::Module::LinkXSetValue(std::string filename) { // Need to check if the document exist // if not we have to create a new one // and Restore the associated Document // std::vector Documents; Documents = App::GetApplication().getDocuments(); for (std::vector::iterator it = Documents.begin(); it != Documents.end(); it++) { if ((*it)->FileName.getValue() == filename) { // The document exist // we can exit return; } } size_t header = filename.find_first_of(":"); string protocol = filename.substr(0, header); string url_new = filename.substr(header + 3); // url_new is the url excluding the http part size_t part2 = url_new.find_first_of("/"); string path = url_new.substr(part2 + 1); // Starting from here the Document doesn't exist we must create it // and make it the active Document before Restoring App::Document* newDoc; string newName; Document* currentDoc = GetApplication().getActiveDocument(); newName = GetApplication().getUniqueDocumentName("unnamed"); newDoc = GetApplication().newDocument(newName.c_str(), (const char*)path.c_str(), true); GetApplication().setActiveDocument(newDoc); this->cloudRestore((const char*)path.c_str()); GetApplication().setActiveDocument(currentDoc); } bool Cloud::Module::cloudRestore(const char* BucketName) { Document* doc = GetApplication().getActiveDocument(); // clean up if the document is not empty // !TODO: mind exceptions while restoring! doc->signalLinkXsetValue.connect(boost::bind(&Cloud::Module::LinkXSetValue, this, _1)); doc->clearUndos(); doc->clearDocument(); std::stringstream oss; Cloud::CloudReader myreader( (const char*)this->URL.getStrValue().c_str(), (const char*)this->TokenAuth.getStrValue().c_str(), (const char*)this->TokenSecret.getStrValue().c_str(), (const char*)this->TCPPort.getStrValue().c_str(), BucketName, (const char*)this->ProtocolVersion.getStrValue().c_str(), this->Region.getStrValue() ); // we shall pass there the initial Document.xml file Base::XMLReader reader("Document.xml", myreader.GetEntry("Document.xml")->FileStream); if (!reader.isValid()) { throw Base::FileException("Error reading Document.xml file", "Document.xml"); } GetApplication().signalStartRestoreDocument(*doc); doc->setStatus(Document::Restoring, true); try { doc->Restore(reader); } catch (const Base::Exception& e) { Base::Console().error("Invalid Document.xml: %s\n", e.what()); } // Special handling for Gui document, the view representations must already // exist, what is done in Restore(). // Note: This file doesn't need to be available if the document has been created // without GUI. But if available then follow after all data files of the App document. doc->signalRestoreDocument(reader); readFiles(myreader, &reader); // reset all touched doc->afterRestore(true); GetApplication().signalFinishRestoreDocument(*doc); doc->setStatus(Document::Restoring, false); doc->Label.setValue(BucketName); // The FileName shall be an URI format string uri; uri = this->URL.getStrValue() + ":" + this->TCPPort.getStrValue() + "/" + string(BucketName); doc->FileName.setValue(uri); doc->signalLinkXsetValue.disconnect(boost::bind(&Cloud::Module::LinkXSetValue, this, _1)); return (true); }