Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix support for non-ASCII characters in Oracle CLOBs #1184

Merged
merged 5 commits into from
Jan 24, 2025
Merged

Fix support for non-ASCII characters in Oracle CLOBs #1184

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d089386
Move table creator helpers to the start of Oracle test
vadz Nov 19, 2024
2b9afba
Make procedure_creator_base helper more flexible and rename it
vadz Nov 19, 2024
5e9f200
Read entire Oracle CLOB and not just its length in characters
vadz Nov 19, 2024
ba202d0
Remove unused offset from OCILobRead() calls
vadz Nov 20, 2024
0db5797
Avoid extra copy of the data in oracle::read_from_lob()
vadz Nov 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions scripts/ci/oracle.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ export ORACLE_SID=XE
# Add path to Oracle libraries.
export LD_LIBRARY_PATH=$ORACLE_HOME/lib

# We need to tell Oracle to use UTF-8 for the tests using non-ASCII strings.
export NLS_LANG=.AL32UTF8

# Execute any command in the Oracle container: pass the command with its
# arguments directly to the function.
oracle_exec()
Expand Down
57 changes: 32 additions & 25 deletions src/backends/oracle/standard-into-type.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -235,41 +235,48 @@ void oracle_standard_into_type_backend::pre_fetch()
void oracle::read_from_lob(oracle_session_backend& session,
OCILobLocator * lobp, std::string & value)
{
ub4 len;

sword res = OCILobGetLength(session.svchp_, session.errhp_,
lobp, &len);
// We can't get the CLOB size in bytes directly, only in characters, which
// is useless for UTF-8 as it doesn't tell us how much memory do we
// actually need for storing it, so we'd have to allocate 4 bytes for every
// character which could be a huge overkill. So instead read the CLOB in
// chunks of its natural size until we get everything.
ub4 len = 0;
sword res = OCILobGetChunkSize(session.svchp_, session.errhp_, lobp, &len);
if (res != OCI_SUCCESS)
{
throw_oracle_soci_error(res, session.errhp_);
}

std::vector<char> buf(len);
value.clear();
vadz marked this conversation as resolved.
Show resolved Hide resolved

if (!len)
return;

if (len != 0)
// Read the LOB in chunks into the buffer while anything remains to be read.
std::vector<char> buf(len);
ub4 offset = 1;
vadz marked this conversation as resolved.
Show resolved Hide resolved
do
{
ub4 lenChunk = len;
ub4 offset = 1;
do
// By setting the input length to 0, we tell Oracle to read as many
// bytes as possible (so called "streaming" mode).
ub4 lenChunk = 0;
res = OCILobRead(session.svchp_, session.errhp_, lobp,
&lenChunk, offset,
&buf[0], len,
0, 0, 0, 0);

if (res == OCI_NEED_DATA)
{
res = OCILobRead(session.svchp_, session.errhp_,
lobp, &lenChunk,
offset,
reinterpret_cast<dvoid*>(&buf[offset - 1]),
len, 0, 0, 0, 0);
if (res == OCI_NEED_DATA)
{
offset += lenChunk;
}
else if (res != OCI_SUCCESS)
{
throw_oracle_soci_error(res, session.errhp_);
}
offset += lenChunk;
}
else if (res != OCI_SUCCESS)
{
throw_oracle_soci_error(res, session.errhp_);
}
while (res == OCI_NEED_DATA);
}

value.assign(buf.begin(), buf.end());
value.append(buf.begin(), buf.begin() + lenChunk);
}
while (res == OCI_NEED_DATA);
}

void oracle_standard_into_type_backend::post_fetch(
Expand Down
216 changes: 135 additions & 81 deletions tests/oracle/test-oracle.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,74 @@ using namespace soci::tests;
std::string connectString;
backend_factory const &backEnd = *soci::factory_oracle();

// Helpers for creating tables for different tests.
struct table_creator_one : public table_creator_base
{
table_creator_one(soci::session & sql)
: table_creator_base(sql)
{
sql << "create table soci_test(id number(10,0), val number(8,0), c char, "
"str varchar2(20), sh number, ll number, ul number, d number, "
"num76 numeric(7,6), "
"tm date, i1 number, i2 number, i3 number, name varchar2(20))";
}
};

struct table_creator_two : public table_creator_base
{
table_creator_two(soci::session & sql)
: table_creator_base(sql)
{
sql << "create table soci_test(num_float number, num_int numeric(4,0),"
" name varchar2(20), sometime date, chr char)";
}
};

struct table_creator_three : public table_creator_base
{
table_creator_three(soci::session & sql)
: table_creator_base(sql)
{
sql << "create table soci_test(name varchar2(100) not null, "
"phone varchar2(15))";
}
};

struct table_creator_four : public table_creator_base
{
table_creator_four(soci::session & sql)
: table_creator_base(sql)
{
sql << "create table soci_test(val number)";
}
};

struct table_creator_for_xml : table_creator_base
{
table_creator_for_xml(soci::session& sql)
: table_creator_base(sql)
{
sql << "create table soci_test(id integer, x xmltype)";
}
};

struct table_creator_for_clob : table_creator_base
{
table_creator_for_clob(soci::session& sql)
: table_creator_base(sql)
{
sql << "create table soci_test(id integer, s clob)";
}
};

struct table_creator_for_blob : public tests::table_creator_base
{
table_creator_for_blob(soci::session &sql) : tests::table_creator_base(sql)
{
sql << "create table soci_test(id integer, b blob)";
}
};

struct table_creator_for_timestamp : public tests::table_creator_base
{
table_creator_for_timestamp(soci::session &sql) : tests::table_creator_base(sql)
Expand Down Expand Up @@ -239,27 +307,30 @@ TEST_CASE("Oracle rowid", "[oracle][rowid]")
}

// Stored procedures
class procedure_creator_base
class creator_base
{
public:
procedure_creator_base(session& sql)
: msession(sql) { drop(); }
explicit creator_base(session& sql, std::string what = "procedure soci_test")
: msession(sql), what_{std::move(what)} { drop(); }

~creator_base() { drop();}

virtual ~procedure_creator_base() { drop();}
private:
void drop()
{
try { msession << "drop procedure soci_test"; } catch (soci_error&) {}
try { msession << "drop " + what_; } catch (soci_error&) {}
}
session& msession;

SOCI_NOT_COPYABLE(procedure_creator_base)
std::string const what_;

SOCI_NOT_COPYABLE(creator_base)
};

struct procedure_creator : procedure_creator_base
struct procedure_creator : creator_base
{
procedure_creator(soci::session & sql)
: procedure_creator_base(sql)
: creator_base(sql)
{
sql <<
"create or replace procedure soci_test(output out varchar2,"
Expand Down Expand Up @@ -325,20 +396,20 @@ namespace soci
};
}

struct in_out_procedure_creator : public procedure_creator_base
struct in_out_procedure_creator : public creator_base
{
in_out_procedure_creator(soci::session & sql)
: procedure_creator_base(sql)
: creator_base(sql)
{
sql << "create or replace procedure soci_test(s in out varchar2)"
" as begin s := s || s; end;";
}
};

struct returns_null_procedure_creator : public procedure_creator_base
struct returns_null_procedure_creator : public creator_base
{
returns_null_procedure_creator(soci::session & sql)
: procedure_creator_base(sql)
: creator_base(sql)
{
sql << "create or replace procedure soci_test(s in out varchar2)"
" as begin s := NULL; end;";
Expand Down Expand Up @@ -774,10 +845,10 @@ struct person_table_creator : public table_creator_base
}
};

struct times100_procedure_creator : public procedure_creator_base
struct times100_procedure_creator : public creator_base
{
times100_procedure_creator(soci::session & sql)
: procedure_creator_base(sql)
: creator_base(sql)
{
sql << "create or replace procedure soci_test(id in out number)"
" as begin id := id * 100; end;";
Expand Down Expand Up @@ -1407,77 +1478,60 @@ TEST_CASE("Bulk iterators", "[oracle][bulkiters]")
sql << "drop table t";
}

TEST_CASE ( "Oracle CLOB", "[oracle][clob]" )
{
soci::session sql ( backEnd, connectString );

// Use a non-ASCII string to test that CLOBs work when byte count differs
// from character count.
//
// Note that this requires some Unicode encoding to be used, e.g. UTF-8 by
// setting NLS_LANG=.AL32UTF8 in the environment.
std::string const test_utf8{"Привет système"};

table_creator_for_clob clob_table(sql);
long_string ls;
ls.value = test_utf8;
sql << "insert into soci_test(id, s) values(1, :s)", use(ls);
ls.value.clear();
sql << "select s from soci_test where id=1", into(ls);
CHECK(ls.value == test_utf8);

// RAII helper to create the function we use below.
struct soci_repeat : creator_base
{
explicit soci_repeat(session& sql)
: creator_base(sql, "function soci_repeat")
{
sql << R"(
create function soci_repeat(s in string, xCount in integer) return clob as
tmp clob;
begin
for i in 1..xCount loop tmp := tmp || s; end loop;
return tmp;
end;
)"
;
}
} soci_repeat_func(sql);

// Append a big number of Unicode chars to the CLOB to test that things
// work with CLOBs larger than a single chunk (which is ~8KiB by default).
//
// Note: Oracle 11 used in the CI tests doesn't seem to handle characters
// outside of the BMP correctly even if current Oracle versions have no
// problems with them, so don't use them here for now.
unsigned xCount = 10000;
sql << "select :s || soci_repeat('Я', :xCount) from dual",
use(ls), use(xCount), into(ls);

REQUIRE(ls.value.length() == test_utf8.length() + 2*xCount);
}

//
// Support for soci Common Tests
//

struct table_creator_one : public table_creator_base
{
table_creator_one(soci::session & sql)
: table_creator_base(sql)
{
sql << "create table soci_test(id number(10,0), val number(8,0), c char, "
"str varchar2(20), sh number, ll number, ul number, d number, "
"num76 numeric(7,6), "
"tm date, i1 number, i2 number, i3 number, name varchar2(20))";
}
};

struct table_creator_two : public table_creator_base
{
table_creator_two(soci::session & sql)
: table_creator_base(sql)
{
sql << "create table soci_test(num_float number, num_int numeric(4,0),"
" name varchar2(20), sometime date, chr char)";
}
};

struct table_creator_three : public table_creator_base
{
table_creator_three(soci::session & sql)
: table_creator_base(sql)
{
sql << "create table soci_test(name varchar2(100) not null, "
"phone varchar2(15))";
}
};

struct table_creator_four : public table_creator_base
{
table_creator_four(soci::session & sql)
: table_creator_base(sql)
{
sql << "create table soci_test(val number)";
}
};

struct table_creator_for_xml : table_creator_base
{
table_creator_for_xml(soci::session& sql)
: table_creator_base(sql)
{
sql << "create table soci_test(id integer, x xmltype)";
}
};

struct table_creator_for_clob : table_creator_base
{
table_creator_for_clob(soci::session& sql)
: table_creator_base(sql)
{
sql << "create table soci_test(id integer, s clob)";
}
};

struct table_creator_for_blob : public tests::table_creator_base
{
table_creator_for_blob(soci::session &sql) : tests::table_creator_base(sql)
{
sql << "create table soci_test(id integer, b blob)";
}
};

class test_context :public test_context_common
{
public:
Expand Down