From 73f99e399d183a9a965239e757fbf86d8c5285e6 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-hx Date: Mon, 7 Apr 2025 17:25:01 -0700 Subject: [PATCH 1/5] POC for Windows --- tests/test_simple_put.cpp | 48 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/test_simple_put.cpp b/tests/test_simple_put.cpp index 615aeef599..c1f29085d3 100755 --- a/tests/test_simple_put.cpp +++ b/tests/test_simple_put.cpp @@ -65,16 +65,60 @@ class StatementPutGetUnicode : public Snowflake::Client::StatementPutGet StatementPutGetUnicode(SF_STMT *stmt) : StatementPutGet(stmt) {} virtual std::string UTF8ToPlatformString(const std::string& utf8_str) { +#ifdef _WIN32 + // convert utf8 string to wstring + auto size_needed = MultiByteToWideChar(CP_UTF8, 0, utf8_str.data(), (int)utf8_str.size(), nullptr, 0); + if (size_needed <= 0) + { + return ""; + } + std::wstring wstr(size_needed, 0); + MultiByteToWideChar(CP_UTF8, 0, utf8_str.data(), (int)utf8_str.size(), wstr.data(), size_needed); + + // convert wstring to platform string + size_needed = WideCharToMultiByte(CP_ACP, 0, wstr.data(), (int)wstr.size(), nullptr, 0, nullptr, nullptr); + if (size_needed <= 0) + { + return ""; + } + std::string result(size_needed, 0); + WideCharToMultiByte(CP_ACP, 0, wstr.data(), (int)wstr.size(), result.data(), size_needed, nullptr, nullptr); + + return result; +#else std::string result = utf8_str; replaceInPlace(result, UTF8_STR, PLATFORM_STR); return result; +#endif } virtual std::string platformStringToUTF8(const std::string& platform_str) { +#ifdef _WIN32 + // convert platform string to wstring + auto size_needed = MultiByteToWideChar(CP_ACP, 0, platform_str.data(), (int)platform_str.size(), nullptr, 0); + if (size_needed <= 0) + { + return ""; + } + std::wstring wstr(size_needed, 0); + MultiByteToWideChar(CP_ACP, 0, platform_str.data(), (int)platform_str.size(), wstr.data(), size_needed); + + // convert wstring to UTF-8 + size_needed = WideCharToMultiByte(CP_UTF8, 0, wstr.data(), (int)wstr.size(), nullptr, 0, nullptr, nullptr); + if (size_needed <= 0) + { + return ""; + } + std::string result(size_needed, 0); + WideCharToMultiByte(CP_UTF8, 0, wstr.data(), (int)wstr.size(), result.data(), size_needed, nullptr, nullptr); + + return result; +#else std::string result = platform_str; replaceInPlace(result, PLATFORM_STR, UTF8_STR); return result; +#endif } }; @@ -1981,6 +2025,7 @@ int main(void) { #endif const struct CMUnitTest tests[] = { +/* cmocka_unit_test_teardown(test_simple_put_auto_compress, teardown), cmocka_unit_test_teardown(test_simple_put_config_temp_dir, teardown), cmocka_unit_test_teardown(test_simple_put_auto_detect_gzip, teardown), @@ -2011,7 +2056,9 @@ int main(void) { cmocka_unit_test_teardown(test_simple_put_with_proxy_fromenv, teardown), cmocka_unit_test_teardown(test_simple_put_with_noproxy_fromenv, teardown), cmocka_unit_test_teardown(test_upload_file_to_stage_using_stream, donothing), +*/ cmocka_unit_test_teardown(test_put_get_with_unicode, teardown), +/* cmocka_unit_test_teardown(test_simple_put_auto_compress_native, teardown), cmocka_unit_test_teardown(test_simple_put_config_temp_dir_native, teardown), cmocka_unit_test_teardown(test_simple_get_native, teardown), @@ -2022,6 +2069,7 @@ int main(void) { cmocka_unit_test_teardown(test_verify_upload, teardown), cmocka_unit_test_teardown(test_simple_put_use_dev_urandom_native, teardown), cmocka_unit_test_teardown(test_simple_put_use_s3_regionalURL_native, teardown), +*/ }; int ret = cmocka_run_group_tests(tests, gr_setup, gr_teardown); return ret; From 652be4f950d5adf52dae6fc8bb5148b883698c9b Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-hx Date: Thu, 17 Apr 2025 23:31:41 +0000 Subject: [PATCH 2/5] add POC on Linux/Mac --- tests/test_simple_put.cpp | 42 +++++++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/tests/test_simple_put.cpp b/tests/test_simple_put.cpp index c1f29085d3..de2904f632 100755 --- a/tests/test_simple_put.cpp +++ b/tests/test_simple_put.cpp @@ -15,6 +15,12 @@ #include "FileTransferAgent.hpp" #include "boost/filesystem.hpp" +#ifndef _WIN32 +#include +#include +#include +#endif + #define COLUMN_STATUS "STATUS" #define COLUMN_SOURCE "SOURCE" #define COLUMN_TARGET "TARGET" @@ -86,8 +92,17 @@ class StatementPutGetUnicode : public Snowflake::Client::StatementPutGet return result; #else - std::string result = utf8_str; - replaceInPlace(result, UTF8_STR, PLATFORM_STR); + setlocale(LC_ALL, ""); + char* encoding = nl_langinfo(CODESET); + iconv_t conv = iconv_open(encoding, "UTF-8"); + std::vector buf(utf8_str.size() * 4 + 1); + char* inbuf = const_cast(utf8_str.data()); + size_t insize = utf8_str.size(); + char* outbuf = const_cast(buf.data()); + size_t outsize = buf.size(); + iconv(conv, &inbuf, &insize, &outbuf, &outsize); + std::string result(buf.data()); + iconv_close(conv); return result; #endif } @@ -115,8 +130,17 @@ class StatementPutGetUnicode : public Snowflake::Client::StatementPutGet return result; #else - std::string result = platform_str; - replaceInPlace(result, PLATFORM_STR, UTF8_STR); + setlocale(LC_ALL, ""); + char* encoding = nl_langinfo(CODESET); + iconv_t conv = iconv_open("UTF-8", encoding); + std::vector buf(platform_str.size() * 4 + 1); + char* inbuf = const_cast(platform_str.data()); + size_t insize = platform_str.size(); + char* outbuf = const_cast(buf.data()); + size_t outsize = buf.size(); + iconv(conv, &inbuf, &insize, &outbuf, &outsize); + std::string result(buf.data()); + iconv_close(conv); return result; #endif } @@ -1970,6 +1994,11 @@ void test_upload_file_to_stage_using_stream(void **unused) void test_put_get_with_unicode(void **unused) { +// On Linux/Mac the default one would be POSIX(ANSI), set to UTF8 to simulate +// application using Unicode. +#ifndef _WIN32 + setlocale(LC_CTYPE, "en_US.utf8"); +#endif std::string dataDir = TestSetup::getDataDir(); std::string filename=PLATFORM_STR + ".csv"; copy_file(dataDir + "small_file.csv", dataDir + filename, copy_options::overwrite_existing); @@ -2025,7 +2054,6 @@ int main(void) { #endif const struct CMUnitTest tests[] = { -/* cmocka_unit_test_teardown(test_simple_put_auto_compress, teardown), cmocka_unit_test_teardown(test_simple_put_config_temp_dir, teardown), cmocka_unit_test_teardown(test_simple_put_auto_detect_gzip, teardown), @@ -2056,9 +2084,7 @@ int main(void) { cmocka_unit_test_teardown(test_simple_put_with_proxy_fromenv, teardown), cmocka_unit_test_teardown(test_simple_put_with_noproxy_fromenv, teardown), cmocka_unit_test_teardown(test_upload_file_to_stage_using_stream, donothing), -*/ cmocka_unit_test_teardown(test_put_get_with_unicode, teardown), -/* cmocka_unit_test_teardown(test_simple_put_auto_compress_native, teardown), cmocka_unit_test_teardown(test_simple_put_config_temp_dir_native, teardown), cmocka_unit_test_teardown(test_simple_get_native, teardown), @@ -2069,8 +2095,8 @@ int main(void) { cmocka_unit_test_teardown(test_verify_upload, teardown), cmocka_unit_test_teardown(test_simple_put_use_dev_urandom_native, teardown), cmocka_unit_test_teardown(test_simple_put_use_s3_regionalURL_native, teardown), -*/ }; int ret = cmocka_run_group_tests(tests, gr_setup, gr_teardown); return ret; } + From 226661ae8faec10589b80d0e12a2711c0c993687 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-hx Date: Mon, 21 Apr 2025 21:14:45 +0000 Subject: [PATCH 3/5] fix test failure --- tests/test_simple_put.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_simple_put.cpp b/tests/test_simple_put.cpp index de2904f632..f81076113a 100755 --- a/tests/test_simple_put.cpp +++ b/tests/test_simple_put.cpp @@ -92,7 +92,6 @@ class StatementPutGetUnicode : public Snowflake::Client::StatementPutGet return result; #else - setlocale(LC_ALL, ""); char* encoding = nl_langinfo(CODESET); iconv_t conv = iconv_open(encoding, "UTF-8"); std::vector buf(utf8_str.size() * 4 + 1); @@ -130,7 +129,6 @@ class StatementPutGetUnicode : public Snowflake::Client::StatementPutGet return result; #else - setlocale(LC_ALL, ""); char* encoding = nl_langinfo(CODESET); iconv_t conv = iconv_open("UTF-8", encoding); std::vector buf(platform_str.size() * 4 + 1); @@ -1997,7 +1995,7 @@ void test_put_get_with_unicode(void **unused) // On Linux/Mac the default one would be POSIX(ANSI), set to UTF8 to simulate // application using Unicode. #ifndef _WIN32 - setlocale(LC_CTYPE, "en_US.utf8"); + setlocale(LC_ALL, "en_US.utf8"); #endif std::string dataDir = TestSetup::getDataDir(); std::string filename=PLATFORM_STR + ".csv"; From 8dae4116042de7012eb1c0b41b1f3f5acc610d55 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-hx Date: Mon, 21 Apr 2025 23:02:25 +0000 Subject: [PATCH 4/5] fix build error on Mac --- tests/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4309717d31..a72e6b52b7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -246,9 +246,9 @@ if (LINUX) pthread -Wl,--no-whole-archive -Wl,--as-needed -static-libgcc -static-libstdc++) endif () if (APPLE) - set(TESTLIB_OPTS_C snowflakeclient ${CMOCKA_LIB} ${ARROW_ALL_LIBS} z m "-framework CoreFoundation -framework SystemConfiguration -framework CoreServices -framework security" -all_load + set(TESTLIB_OPTS_C snowflakeclient ${CMOCKA_LIB} ${ARROW_ALL_LIBS} z m iconv "-framework CoreFoundation -framework SystemConfiguration -framework CoreServices -framework security" -all_load ${OOB_LIB} ${CURL_LIB} ${SSL_LIB} ${CRYPTO_LIB} pthread ${AZURE_STORAGE_LITE_LIB} ${AWS_ALL_LIBS} ${UUID_LIB}) - set(TESTLIB_OPTS_CXX snowflakeclient ${CMOCKA_LIB} ${ARROW_ALL_LIBS} z m "-framework CoreFoundation -framework SystemConfiguration -framework CoreServices -framework security" -all_load ${CURL_LIB} + set(TESTLIB_OPTS_CXX snowflakeclient ${CMOCKA_LIB} ${ARROW_ALL_LIBS} z m iconv "-framework CoreFoundation -framework SystemConfiguration -framework CoreServices -framework security" -all_load ${CURL_LIB} ${OOB_LIB} ${SSL_LIB} ${CRYPTO_LIB} pthread ${AZURE_STORAGE_LITE_LIB} ${AWS_ALL_LIBS} ${UUID_LIB}) endif() if (WIN32) From 836bf320fdbdf17c7bc869d9807a775272e1a01d Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-hx Date: Mon, 21 Apr 2025 23:46:02 +0000 Subject: [PATCH 5/5] fix test failure on Mac --- tests/test_simple_put.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_simple_put.cpp b/tests/test_simple_put.cpp index f81076113a..015c89a133 100755 --- a/tests/test_simple_put.cpp +++ b/tests/test_simple_put.cpp @@ -1994,8 +1994,10 @@ void test_put_get_with_unicode(void **unused) { // On Linux/Mac the default one would be POSIX(ANSI), set to UTF8 to simulate // application using Unicode. -#ifndef _WIN32 +#ifdef __linux__ setlocale(LC_ALL, "en_US.utf8"); +#elif defined(__APPLE__) + setlocale(LC_ALL, "en_US.UTF-8"); #endif std::string dataDir = TestSetup::getDataDir(); std::string filename=PLATFORM_STR + ".csv";