Skip to content

SNOW-1737939 unicode support POC #878

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 5 commits into
base: master
Choose a base branch
from
+80 −6
Draft

SNOW-1737939 unicode support POC #878

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
73f99e3
POC for Windows
sfc-gh-ext-simba-hx Apr 8, 2025
652be4f
add POC on Linux/Mac
sfc-gh-ext-simba-hx Apr 17, 2025
226661a
fix test failure
sfc-gh-ext-simba-hx Apr 21, 2025
8dae411
fix build error on Mac
sfc-gh-ext-simba-hx Apr 21, 2025
836bf32
fix test failure on Mac
sfc-gh-ext-simba-hx Apr 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions tests/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -246,9 +246,9 @@ if (LINUX)
pthread -Wl,--no-whole-archive -Wl,--as-needed -static-libgcc -static-libstdc++)
endif ()
if (APPLE)
set(TESTLIB_OPTS_C snowflakeclient ${CMOCKA_LIB} ${ARROW_ALL_LIBS} z m "-framework CoreFoundation -framework SystemConfiguration -framework CoreServices -framework security" -all_load
set(TESTLIB_OPTS_C snowflakeclient ${CMOCKA_LIB} ${ARROW_ALL_LIBS} z m iconv "-framework CoreFoundation -framework SystemConfiguration -framework CoreServices -framework security" -all_load
${OOB_LIB} ${CURL_LIB} ${SSL_LIB} ${CRYPTO_LIB} pthread ${AZURE_STORAGE_LITE_LIB} ${AWS_ALL_LIBS} ${UUID_LIB})
set(TESTLIB_OPTS_CXX snowflakeclient ${CMOCKA_LIB} ${ARROW_ALL_LIBS} z m "-framework CoreFoundation -framework SystemConfiguration -framework CoreServices -framework security" -all_load ${CURL_LIB}
set(TESTLIB_OPTS_CXX snowflakeclient ${CMOCKA_LIB} ${ARROW_ALL_LIBS} z m iconv "-framework CoreFoundation -framework SystemConfiguration -framework CoreServices -framework security" -all_load ${CURL_LIB}
${OOB_LIB} ${SSL_LIB} ${CRYPTO_LIB} pthread ${AZURE_STORAGE_LITE_LIB} ${AWS_ALL_LIBS} ${UUID_LIB})
endif()
if (WIN32)
Expand Down
82 changes: 78 additions & 4 deletions tests/test_simple_put.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,12 @@
#include "FileTransferAgent.hpp"
#include "boost/filesystem.hpp"

#ifndef _WIN32
#include <locale.h>
#include <langinfo.h>
#include <iconv.h>
#endif

#define COLUMN_STATUS "STATUS"
#define COLUMN_SOURCE "SOURCE"
#define COLUMN_TARGET "TARGET"
Expand Down Expand Up @@ -65,16 +71,76 @@ class StatementPutGetUnicode : public Snowflake::Client::StatementPutGet
StatementPutGetUnicode(SF_STMT *stmt) : StatementPutGet(stmt) {}
virtual std::string UTF8ToPlatformString(const std::string& utf8_str)
{
std::string result = utf8_str;
replaceInPlace(result, UTF8_STR, PLATFORM_STR);
#ifdef _WIN32
// convert utf8 string to wstring
auto size_needed = MultiByteToWideChar(CP_UTF8, 0, utf8_str.data(), (int)utf8_str.size(), nullptr, 0);
if (size_needed <= 0)
{
return "";
}
std::wstring wstr(size_needed, 0);
MultiByteToWideChar(CP_UTF8, 0, utf8_str.data(), (int)utf8_str.size(), wstr.data(), size_needed);

// convert wstring to platform string
size_needed = WideCharToMultiByte(CP_ACP, 0, wstr.data(), (int)wstr.size(), nullptr, 0, nullptr, nullptr);
if (size_needed <= 0)
{
return "";
}
std::string result(size_needed, 0);
WideCharToMultiByte(CP_ACP, 0, wstr.data(), (int)wstr.size(), result.data(), size_needed, nullptr, nullptr);

return result;
#else
char* encoding = nl_langinfo(CODESET);
iconv_t conv = iconv_open(encoding, "UTF-8");
std::vector<char> buf(utf8_str.size() * 4 + 1);
char* inbuf = const_cast<char*>(utf8_str.data());
size_t insize = utf8_str.size();
char* outbuf = const_cast<char*>(buf.data());
size_t outsize = buf.size();
iconv(conv, &inbuf, &insize, &outbuf, &outsize);
std::string result(buf.data());
iconv_close(conv);
return result;
#endif
}

virtual std::string platformStringToUTF8(const std::string& platform_str)
{
std::string result = platform_str;
replaceInPlace(result, PLATFORM_STR, UTF8_STR);
#ifdef _WIN32
// convert platform string to wstring
auto size_needed = MultiByteToWideChar(CP_ACP, 0, platform_str.data(), (int)platform_str.size(), nullptr, 0);
if (size_needed <= 0)
{
return "";
}
std::wstring wstr(size_needed, 0);
MultiByteToWideChar(CP_ACP, 0, platform_str.data(), (int)platform_str.size(), wstr.data(), size_needed);

// convert wstring to UTF-8
size_needed = WideCharToMultiByte(CP_UTF8, 0, wstr.data(), (int)wstr.size(), nullptr, 0, nullptr, nullptr);
if (size_needed <= 0)
{
return "";
}
std::string result(size_needed, 0);
WideCharToMultiByte(CP_UTF8, 0, wstr.data(), (int)wstr.size(), result.data(), size_needed, nullptr, nullptr);

return result;
#else
char* encoding = nl_langinfo(CODESET);
iconv_t conv = iconv_open("UTF-8", encoding);
std::vector<char> buf(platform_str.size() * 4 + 1);
char* inbuf = const_cast<char*>(platform_str.data());
size_t insize = platform_str.size();
char* outbuf = const_cast<char*>(buf.data());
size_t outsize = buf.size();
iconv(conv, &inbuf, &insize, &outbuf, &outsize);
std::string result(buf.data());
iconv_close(conv);
return result;
#endif
}
};

Expand Down Expand Up @@ -1926,6 +1992,13 @@ void test_upload_file_to_stage_using_stream(void **unused)

void test_put_get_with_unicode(void **unused)
{
// On Linux/Mac the default one would be POSIX(ANSI), set to UTF8 to simulate
// application using Unicode.
#ifdef __linux__
setlocale(LC_ALL, "en_US.utf8");
#elif defined(__APPLE__)
setlocale(LC_ALL, "en_US.UTF-8");
#endif
std::string dataDir = TestSetup::getDataDir();
std::string filename=PLATFORM_STR + ".csv";
copy_file(dataDir + "small_file.csv", dataDir + filename, copy_options::overwrite_existing);
Expand Down Expand Up @@ -2026,3 +2099,4 @@ int main(void) {
int ret = cmocka_run_group_tests(tests, gr_setup, gr_teardown);
return ret;
}