mbutils.c File Reference

#include "postgres.h"
#include "access/xact.h"
#include "catalog/namespace.h"
#include "mb/pg_wchar.h"
#include "utils/fmgrprotos.h"
#include "utils/memutils.h"
#include "utils/relcache.h"
#include "varatt.h"

Include dependency graph for mbutils.c:

Go to the source code of this file.

Data Structures
struct	ConvProcInfo

Typedefs
typedef struct ConvProcInfo	ConvProcInfo

Functions
static char *	perform_default_encoding_conversion (const char *src, int len, bool is_client_to_server)
static int	cliplen (const char *str, int len, int limit)
int	PrepareClientEncoding (int encoding)
int	SetClientEncoding (int encoding)
void	InitializeClientEncoding (void)
int	pg_get_client_encoding (void)
const char *	pg_get_client_encoding_name (void)
unsigned char *	pg_do_encoding_conversion (unsigned char *src, int len, int src_encoding, int dest_encoding)
int	pg_do_encoding_conversion_buf (Oid proc, int src_encoding, int dest_encoding, unsigned char *src, int srclen, unsigned char *dest, int destlen, bool noError)
Datum	pg_convert_to (PG_FUNCTION_ARGS)
Datum	pg_convert_from (PG_FUNCTION_ARGS)
Datum	pg_convert (PG_FUNCTION_ARGS)
Datum	length_in_encoding (PG_FUNCTION_ARGS)
Datum	pg_encoding_max_length_sql (PG_FUNCTION_ARGS)
char *	pg_client_to_server (const char *s, int len)
char *	pg_any_to_server (const char *s, int len, int encoding)
char *	pg_server_to_client (const char *s, int len)
char *	pg_server_to_any (const char *s, int len, int encoding)
void	pg_unicode_to_server (pg_wchar c, unsigned char *s)
bool	pg_unicode_to_server_noerror (pg_wchar c, unsigned char *s)
int	pg_mb2wchar (const char *from, pg_wchar *to)
int	pg_mb2wchar_with_len (const char *from, pg_wchar *to, int len)
int	pg_encoding_mb2wchar_with_len (int encoding, const char *from, pg_wchar *to, int len)
int	pg_wchar2mb (const pg_wchar *from, char *to)
int	pg_wchar2mb_with_len (const pg_wchar *from, char *to, int len)
int	pg_encoding_wchar2mb_with_len (int encoding, const pg_wchar *from, char *to, int len)
int	pg_mblen (const char *mbstr)
int	pg_dsplen (const char *mbstr)
int	pg_mbstrlen (const char *mbstr)
int	pg_mbstrlen_with_len (const char *mbstr, int limit)
int	pg_mbcliplen (const char *mbstr, int len, int limit)
int	pg_encoding_mbcliplen (int encoding, const char *mbstr, int len, int limit)
int	pg_mbcharcliplen (const char *mbstr, int len, int limit)
void	SetDatabaseEncoding (int encoding)
void	SetMessageEncoding (int encoding)
int	GetDatabaseEncoding (void)
const char *	GetDatabaseEncodingName (void)
Datum	getdatabaseencoding (PG_FUNCTION_ARGS)
Datum	pg_client_encoding (PG_FUNCTION_ARGS)
Datum	PG_char_to_encoding (PG_FUNCTION_ARGS)
Datum	PG_encoding_to_char (PG_FUNCTION_ARGS)
int	GetMessageEncoding (void)
static bool	pg_generic_charinc (unsigned char *charptr, int len)
static bool	pg_utf8_increment (unsigned char *charptr, int length)
static bool	pg_eucjp_increment (unsigned char *charptr, int length)
mbcharacter_incrementer	pg_database_encoding_character_incrementer (void)
int	pg_database_encoding_max_length (void)
bool	pg_verifymbstr (const char *mbstr, int len, bool noError)
bool	pg_verify_mbstr (int encoding, const char *mbstr, int len, bool noError)
int	pg_verify_mbstr_len (int encoding, const char *mbstr, int len, bool noError)
void	check_encoding_conversion_args (int src_encoding, int dest_encoding, int len, int expected_src_encoding, int expected_dest_encoding)
void	report_invalid_encoding (int encoding, const char *mbstr, int len)
void	report_untranslatable_char (int src_encoding, int dest_encoding, const char *mbstr, int len)

Variables
static List *	ConvProcList = NIL
static FmgrInfo *	ToServerConvProc = NULL
static FmgrInfo *	ToClientConvProc = NULL
static FmgrInfo *	Utf8ToServerConvProc = NULL
static const pg_enc2name *	ClientEncoding = &pg_enc2name_tbl[PG_SQL_ASCII]
static const pg_enc2name *	DatabaseEncoding = &pg_enc2name_tbl[PG_SQL_ASCII]
static const pg_enc2name *	MessageEncoding = &pg_enc2name_tbl[PG_SQL_ASCII]
static bool	backend_startup_complete = false
static int	pending_client_encoding = PG_SQL_ASCII

Typedef Documentation

ConvProcInfo

typedef struct ConvProcInfo ConvProcInfo

Function Documentation

check_encoding_conversion_args()

void check_encoding_conversion_args	(	int	src_encoding,
		int	dest_encoding,
		int	len,
		int	expected_src_encoding,
		int	expected_dest_encoding
	)

Definition at line 1670 of file mbutils.c.

{
    if (!PG_VALID_ENCODING(src_encoding))
        elog(ERROR, "invalid source encoding ID: %d", src_encoding);
    if (src_encoding != expected_src_encoding && expected_src_encoding >= 0)
        elog(ERROR, "expected source encoding \"%s\", but got \"%s\"",
             pg_enc2name_tbl[expected_src_encoding].name,
             pg_enc2name_tbl[src_encoding].name);
    if (!PG_VALID_ENCODING(dest_encoding))
        elog(ERROR, "invalid destination encoding ID: %d", dest_encoding);
    if (dest_encoding != expected_dest_encoding && expected_dest_encoding >= 0)
        elog(ERROR, "expected destination encoding \"%s\", but got \"%s\"",
             pg_enc2name_tbl[expected_dest_encoding].name,
             pg_enc2name_tbl[dest_encoding].name);
    if (len < 0)
        elog(ERROR, "encoding conversion length must not be negative");
}

References elog, ERROR, len, name, pg_enc2name_tbl, and PG_VALID_ENCODING.

cliplen()

static int cliplen ( const char *  str, int  len, int  limit  )

static

Definition at line 1151 of file mbutils.c.

{
    int         l = 0;
 
    len = Min(len, limit);
    while (l < len && str[l])
        l++;
    return l;
}

References len, Min, and str.

Referenced by pg_encoding_mbcliplen(), pg_mbcharcliplen(), and pgstat_clip_activity().

getdatabaseencoding()

Datum getdatabaseencoding

(

PG_FUNCTION_ARGS

)

Definition at line 1274 of file mbutils.c.

{
    return DirectFunctionCall1(namein, CStringGetDatum(DatabaseEncoding->name));
}

References CStringGetDatum(), DatabaseEncoding, DirectFunctionCall1, pg_enc2name::name, and namein().

GetDatabaseEncoding()

int GetDatabaseEncoding

(

void

)

Definition at line 1262 of file mbutils.c.

{
    return DatabaseEncoding->encoding;
}

References DatabaseEncoding, and pg_enc2name::encoding.

Referenced by ascii(), BeginCopyFrom(), BeginCopyTo(), char2wchar(), chr(), CollationCreate(), CollationGetCollid(), compareStrings(), convert_from_utf8(), convert_to_utf8(), CopyConversionError(), CopyConvertBuf(), create_pg_locale_builtin(), create_pg_locale_icu(), create_pg_locale_libc(), cstr2sv(), dblink_connect(), dblink_get_conn(), DefineCollation(), Generic_Text_IC_like(), GenericMatchText(), get_collation_oid(), get_json_object_as_hash(), InitializeClientEncoding(), IsThereCollationInNamespace(), json_recv(), jsonb_from_cstring(), locate_stem_module(), LogicalOutputWrite(), makeJsonLexContext(), p_isspecial(), ParallelWorkerMain(), pg_database_encoding_character_incrementer(), pg_database_encoding_max_length(), pg_generic_charinc(), pg_perm_setlocale(), pg_set_regex_collation(), pg_unicode_to_server(), pg_unicode_to_server_noerror(), pg_verifymbstr(), pgss_store(), PLyUnicode_Bytes(), populate_array_json(), PrepareClientEncoding(), read_extension_script_file(), SetClientEncoding(), str_casefold(), sv2cstr(), text_position_setup(), to_ascii_default(), type_maximum_size(), unicode_assigned(), unicode_norm_form_from_string(), wchar2char(), xml_in(), xml_is_document(), xmlparse(), and xmltotext_with_options().

GetDatabaseEncodingName()

const char * GetDatabaseEncodingName

(

void

)

Definition at line 1268 of file mbutils.c.

{
    return DatabaseEncoding->name;
}

References DatabaseEncoding, and pg_enc2name::name.

Referenced by check_client_encoding(), CheckMyDatabase(), connect_pg_server(), dblink_connect(), dblink_get_conn(), get_collation_oid(), InitializeClientEncoding(), IsThereCollationInNamespace(), json_errdetail(), libpqrcv_connect(), locate_stem_module(), pg_unicode_to_server(), ProcessConfigFileInternal(), and regcollationin().

GetMessageEncoding()

int GetMessageEncoding

(

void

)

Definition at line 1309 of file mbutils.c.

{
    return MessageEncoding->encoding;
}

References pg_enc2name::encoding, and MessageEncoding.

InitializeClientEncoding()

void InitializeClientEncoding

(

void

)

Definition at line 282 of file mbutils.c.

{
    int         current_server_encoding;
 
    Assert(!backend_startup_complete);
    backend_startup_complete = true;
 
    if (PrepareClientEncoding(pending_client_encoding) < 0 ||
        SetClientEncoding(pending_client_encoding) < 0)
    {
        /*
         * Oops, the requested conversion is not available. We couldn't fail
         * before, but we can now.
         */
        ereport(FATAL,
                (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
                 errmsg("conversion between %s and %s is not supported",
                        pg_enc2name_tbl[pending_client_encoding].name,
                        GetDatabaseEncodingName())));
    }
 
    /*
     * Also look up the UTF8-to-server conversion function if needed.  Since
     * the server encoding is fixed within any one backend process, we don't
     * have to do this more than once.
     */
    current_server_encoding = GetDatabaseEncoding();
    if (current_server_encoding != PG_UTF8 &&
        current_server_encoding != PG_SQL_ASCII)
    {
        Oid         utf8_to_server_proc;
 
        AssertCouldGetRelation();
        utf8_to_server_proc =
            FindDefaultConversionProc(PG_UTF8,
                                      current_server_encoding);
        /* If there's no such conversion, just leave the pointer as NULL */
        if (OidIsValid(utf8_to_server_proc))
        {
            FmgrInfo   *finfo;
 
            finfo = (FmgrInfo *) MemoryContextAlloc(TopMemoryContext,
                                                    sizeof(FmgrInfo));
            fmgr_info_cxt(utf8_to_server_proc, finfo,
                          TopMemoryContext);
            /* Set Utf8ToServerConvProc only after data is fully valid */
            Utf8ToServerConvProc = finfo;
        }
    }
}

References Assert(), AssertCouldGetRelation(), backend_startup_complete, ereport, errcode(), errmsg(), FATAL, FindDefaultConversionProc(), fmgr_info_cxt(), GetDatabaseEncoding(), GetDatabaseEncodingName(), MemoryContextAlloc(), name, OidIsValid, pending_client_encoding, pg_enc2name_tbl, PG_SQL_ASCII, PG_UTF8, PrepareClientEncoding(), SetClientEncoding(), TopMemoryContext, and Utf8ToServerConvProc.

Referenced by InitPostgres().

length_in_encoding()

Datum length_in_encoding

(

PG_FUNCTION_ARGS

)

Definition at line 616 of file mbutils.c.

{
    bytea      *string = PG_GETARG_BYTEA_PP(0);
    char       *src_encoding_name = NameStr(*PG_GETARG_NAME(1));
    int         src_encoding = pg_char_to_encoding(src_encoding_name);
    const char *src_str;
    int         len;
    int         retval;
 
    if (src_encoding < 0)
        ereport(ERROR,
                (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
                 errmsg("invalid encoding name \"%s\"",
                        src_encoding_name)));
 
    len = VARSIZE_ANY_EXHDR(string);
    src_str = VARDATA_ANY(string);
 
    retval = pg_verify_mbstr_len(src_encoding, src_str, len, false);
 
    PG_RETURN_INT32(retval);
}

References ereport, errcode(), errmsg(), ERROR, len, NameStr, pg_char_to_encoding, PG_GETARG_BYTEA_PP, PG_GETARG_NAME, PG_RETURN_INT32, pg_verify_mbstr_len(), VARDATA_ANY, and VARSIZE_ANY_EXHDR.

perform_default_encoding_conversion()

static char * perform_default_encoding_conversion ( const char *  src, int  len, bool  is_client_to_server  )

static

Definition at line 784 of file mbutils.c.

{
    char       *result;
    int         src_encoding,
                dest_encoding;
    FmgrInfo   *flinfo;
 
    if (is_client_to_server)
    {
        src_encoding = ClientEncoding->encoding;
        dest_encoding = DatabaseEncoding->encoding;
        flinfo = ToServerConvProc;
    }
    else
    {
        src_encoding = DatabaseEncoding->encoding;
        dest_encoding = ClientEncoding->encoding;
        flinfo = ToClientConvProc;
    }
 
    if (flinfo == NULL)
        return unconstify(char *, src);
 
    /*
     * Allocate space for conversion result, being wary of integer overflow.
     * See comments in pg_do_encoding_conversion.
     */
    if ((Size) len >= (MaxAllocHugeSize / (Size) MAX_CONVERSION_GROWTH))
        ereport(ERROR,
                (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
                 errmsg("out of memory"),
                 errdetail("String of %d bytes is too long for encoding conversion.",
                           len)));
 
    result = (char *)
        MemoryContextAllocHuge(CurrentMemoryContext,
                               (Size) len * MAX_CONVERSION_GROWTH + 1);
 
    FunctionCall6(flinfo,
                  Int32GetDatum(src_encoding),
                  Int32GetDatum(dest_encoding),
                  CStringGetDatum(src),
                  CStringGetDatum(result),
                  Int32GetDatum(len),
                  BoolGetDatum(false));
 
    /*
     * Release extra space if there might be a lot --- see comments in
     * pg_do_encoding_conversion.
     */
    if (len > 1000000)
    {
        Size        resultlen = strlen(result);
 
        if (resultlen >= MaxAllocSize)
            ereport(ERROR,
                    (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
                     errmsg("out of memory"),
                     errdetail("String of %d bytes is too long for encoding conversion.",
                               len)));
 
        result = (char *) repalloc(result, resultlen + 1);
    }
 
    return result;
}

References BoolGetDatum(), ClientEncoding, CStringGetDatum(), CurrentMemoryContext, DatabaseEncoding, pg_enc2name::encoding, ereport, errcode(), errdetail(), errmsg(), ERROR, FunctionCall6, Int32GetDatum(), len, MAX_CONVERSION_GROWTH, MaxAllocHugeSize, MaxAllocSize, MemoryContextAllocHuge(), repalloc(), ToClientConvProc, ToServerConvProc, and unconstify.

Referenced by pg_any_to_server(), and pg_server_to_any().

pg_any_to_server()

char * pg_any_to_server	(	const char *	s,
		int	len,
		int	encoding
	)

Definition at line 677 of file mbutils.c.

{
    if (len <= 0)
        return unconstify(char *, s);   /* empty string is always valid */
 
    if (encoding == DatabaseEncoding->encoding ||
        encoding == PG_SQL_ASCII)
    {
        /*
         * No conversion is needed, but we must still validate the data.
         */
        (void) pg_verify_mbstr(DatabaseEncoding->encoding, s, len, false);
        return unconstify(char *, s);
    }
 
    if (DatabaseEncoding->encoding == PG_SQL_ASCII)
    {
        /*
         * No conversion is possible, but we must still validate the data,
         * because the client-side code might have done string escaping using
         * the selected client_encoding.  If the client encoding is ASCII-safe
         * then we just do a straight validation under that encoding.  For an
         * ASCII-unsafe encoding we have a problem: we dare not pass such data
         * to the parser but we have no way to convert it.  We compromise by
         * rejecting the data if it contains any non-ASCII characters.
         */
        if (PG_VALID_BE_ENCODING(encoding))
            (void) pg_verify_mbstr(encoding, s, len, false);
        else
        {
            int         i;
 
            for (i = 0; i < len; i++)
            {
                if (s[i] == '\0' || IS_HIGHBIT_SET(s[i]))
                    ereport(ERROR,
                            (errcode(ERRCODE_CHARACTER_NOT_IN_REPERTOIRE),
                             errmsg("invalid byte value for encoding \"%s\": 0x%02x",
                                    pg_enc2name_tbl[PG_SQL_ASCII].name,
                                    (unsigned char) s[i])));
            }
        }
        return unconstify(char *, s);
    }
 
    /* Fast path if we can use cached conversion function */
    if (encoding == ClientEncoding->encoding)
        return perform_default_encoding_conversion(s, len, true);
 
    /* General case ... will not work outside transactions */
    return (char *) pg_do_encoding_conversion((unsigned char *) unconstify(char *, s),
                                              len,
                                              encoding,
                                              DatabaseEncoding->encoding);
}

References ClientEncoding, DatabaseEncoding, encoding, pg_enc2name::encoding, ereport, errcode(), errmsg(), ERROR, i, IS_HIGHBIT_SET, len, name, perform_default_encoding_conversion(), pg_do_encoding_conversion(), pg_enc2name_tbl, PG_SQL_ASCII, PG_VALID_BE_ENCODING, pg_verify_mbstr(), and unconstify.

Referenced by ASN1_STRING_to_text(), cache_single_string(), db_encoding_convert(), dsnowball_lexize(), pg_client_to_server(), pg_stat_statements_internal(), pgp_armor_headers(), PLyUnicode_Bytes(), read_extension_script_file(), tsearch_readline(), utf_u2e(), X509_NAME_to_cstring(), and xml_recv().

PG_char_to_encoding()

Datum PG_char_to_encoding

(

PG_FUNCTION_ARGS

)

Definition at line 1286 of file mbutils.c.

{
    Name        s = PG_GETARG_NAME(0);
 
    PG_RETURN_INT32(pg_char_to_encoding(NameStr(*s)));
}

References NameStr, pg_char_to_encoding, PG_GETARG_NAME, and PG_RETURN_INT32.

pg_client_encoding()

Datum pg_client_encoding

(

PG_FUNCTION_ARGS

)

Definition at line 1280 of file mbutils.c.

{
    return DirectFunctionCall1(namein, CStringGetDatum(ClientEncoding->name));
}

References ClientEncoding, CStringGetDatum(), DirectFunctionCall1, pg_enc2name::name, and namein().

pg_client_to_server()

char * pg_client_to_server	(	const char *	s,
		int	len
	)

Definition at line 661 of file mbutils.c.

{
    return pg_any_to_server(s, len, ClientEncoding->encoding);
}

References ClientEncoding, pg_enc2name::encoding, len, and pg_any_to_server().

Referenced by exec_bind_message(), parse_fcall_arguments(), pq_getmsgstring(), and pq_getmsgtext().

pg_convert()

Datum pg_convert

(

PG_FUNCTION_ARGS

)

Definition at line 554 of file mbutils.c.

{
    bytea      *string = PG_GETARG_BYTEA_PP(0);
    char       *src_encoding_name = NameStr(*PG_GETARG_NAME(1));
    int         src_encoding = pg_char_to_encoding(src_encoding_name);
    char       *dest_encoding_name = NameStr(*PG_GETARG_NAME(2));
    int         dest_encoding = pg_char_to_encoding(dest_encoding_name);
    const char *src_str;
    char       *dest_str;
    bytea      *retval;
    int         len;
 
    if (src_encoding < 0)
        ereport(ERROR,
                (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
                 errmsg("invalid source encoding name \"%s\"",
                        src_encoding_name)));
    if (dest_encoding < 0)
        ereport(ERROR,
                (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
                 errmsg("invalid destination encoding name \"%s\"",
                        dest_encoding_name)));
 
    /* make sure that source string is valid */
    len = VARSIZE_ANY_EXHDR(string);
    src_str = VARDATA_ANY(string);
    (void) pg_verify_mbstr(src_encoding, src_str, len, false);
 
    /* perform conversion */
    dest_str = (char *) pg_do_encoding_conversion((unsigned char *) unconstify(char *, src_str),
                                                  len,
                                                  src_encoding,
                                                  dest_encoding);
 
 
    /* return source string if no conversion happened */
    if (dest_str == src_str)
        PG_RETURN_BYTEA_P(string);
 
    /*
     * build bytea data type structure.
     */
    len = strlen(dest_str);
    retval = (bytea *) palloc(len + VARHDRSZ);
    SET_VARSIZE(retval, len + VARHDRSZ);
    memcpy(VARDATA(retval), dest_str, len);
    pfree(dest_str);
 
    /* free memory if allocated by the toaster */
    PG_FREE_IF_COPY(string, 0);
 
    PG_RETURN_BYTEA_P(retval);
}

References ereport, errcode(), errmsg(), ERROR, len, NameStr, palloc(), pfree(), pg_char_to_encoding, pg_do_encoding_conversion(), PG_FREE_IF_COPY, PG_GETARG_BYTEA_PP, PG_GETARG_NAME, PG_RETURN_BYTEA_P, pg_verify_mbstr(), SET_VARSIZE, unconstify, VARDATA, VARDATA_ANY, VARHDRSZ, and VARSIZE_ANY_EXHDR.

Referenced by pg_convert_from(), and pg_convert_to().

pg_convert_from()

Datum pg_convert_from

(

PG_FUNCTION_ARGS

)

Definition at line 527 of file mbutils.c.

{
    Datum       string = PG_GETARG_DATUM(0);
    Datum       src_encoding_name = PG_GETARG_DATUM(1);
    Datum       dest_encoding_name = DirectFunctionCall1(namein,
                                                         CStringGetDatum(DatabaseEncoding->name));
    Datum       result;
 
    result = DirectFunctionCall3(pg_convert, string,
                                 src_encoding_name, dest_encoding_name);
 
    /*
     * pg_convert returns a bytea, which we in turn return as text, relying on
     * the fact that they are both in fact varlena types, and thus
     * structurally identical. Although not all bytea values are valid text,
     * in this case it will be because we've told pg_convert to return one
     * that is valid as text in the current database encoding.
     */
    PG_RETURN_DATUM(result);
}

References CStringGetDatum(), DatabaseEncoding, DirectFunctionCall1, DirectFunctionCall3, pg_enc2name::name, namein(), pg_convert(), PG_GETARG_DATUM, and PG_RETURN_DATUM.

pg_convert_to()

Datum pg_convert_to

(

PG_FUNCTION_ARGS

)

Definition at line 502 of file mbutils.c.

{
    Datum       string = PG_GETARG_DATUM(0);
    Datum       dest_encoding_name = PG_GETARG_DATUM(1);
    Datum       src_encoding_name = DirectFunctionCall1(namein,
                                                        CStringGetDatum(DatabaseEncoding->name));
    Datum       result;
 
    /*
     * pg_convert expects a bytea as its first argument. We're passing it a
     * text argument here, relying on the fact that they are both in fact
     * varlena types, and thus structurally identical.
     */
    result = DirectFunctionCall3(pg_convert, string,
                                 src_encoding_name, dest_encoding_name);
 
    PG_RETURN_DATUM(result);
}

References CStringGetDatum(), DatabaseEncoding, DirectFunctionCall1, DirectFunctionCall3, pg_enc2name::name, namein(), pg_convert(), PG_GETARG_DATUM, and PG_RETURN_DATUM.

pg_database_encoding_character_incrementer()

mbcharacter_incrementer pg_database_encoding_character_incrementer

(

void

)

Definition at line 1524 of file mbutils.c.

{
    /*
     * Eventually it might be best to add a field to pg_wchar_table[], but for
     * now we just use a switch.
     */
    switch (GetDatabaseEncoding())
    {
        case PG_UTF8:
            return pg_utf8_increment;
 
        case PG_EUC_JP:
            return pg_eucjp_increment;
 
        default:
            return pg_generic_charinc;
    }
}

References GetDatabaseEncoding(), PG_EUC_JP, pg_eucjp_increment(), pg_generic_charinc(), PG_UTF8, and pg_utf8_increment().

Referenced by make_greater_string().

pg_database_encoding_max_length()

int pg_database_encoding_max_length

(

void

)

Definition at line 1547 of file mbutils.c.

{
    return pg_wchar_table[GetDatabaseEncoding()].maxmblen;
}

References GetDatabaseEncoding(), pg_wchar_tbl::maxmblen, and pg_wchar_table.

Referenced by bpcharlen(), charlen_to_bytelen(), dotrim(), downcase_identifier(), gbt_bpchar_consistent(), gbt_text_compress(), gbt_text_consistent(), generate_trgm_only(), Generic_Text_IC_like(), GenericMatchText(), infix(), init_tsvector_parser(), like_escape(), like_fixed_prefix(), lpad(), pg_mbcharcliplen(), pg_mbstrlen(), pg_mbstrlen_with_len(), protect_out_of_mem(), regexp_fixed_prefix(), rpad(), setup_regexp_matches(), setup_test_matches(), show_trgm(), strlower_libc(), strlower_libc_mb(), strtitle_libc(), strtitle_libc_mb(), strupper_libc(), strupper_libc_mb(), text_length(), text_position_setup(), text_reverse(), text_substring(), TParserInit(), translate(), and tsvectorout().

pg_do_encoding_conversion()

unsigned char * pg_do_encoding_conversion	(	unsigned char *	src,
		int	len,
		int	src_encoding,
		int	dest_encoding
	)

Definition at line 357 of file mbutils.c.

{
    unsigned char *result;
    Oid         proc;
 
    if (len <= 0)
        return src;             /* empty string is always valid */
 
    if (src_encoding == dest_encoding)
        return src;             /* no conversion required, assume valid */
 
    if (dest_encoding == PG_SQL_ASCII)
        return src;             /* any string is valid in SQL_ASCII */
 
    if (src_encoding == PG_SQL_ASCII)
    {
        /* No conversion is possible, but we must validate the result */
        (void) pg_verify_mbstr(dest_encoding, (const char *) src, len, false);
        return src;
    }
 
    if (!IsTransactionState())  /* shouldn't happen */
        elog(ERROR, "cannot perform encoding conversion outside a transaction");
 
    proc = FindDefaultConversionProc(src_encoding, dest_encoding);
    if (!OidIsValid(proc))
        ereport(ERROR,
                (errcode(ERRCODE_UNDEFINED_FUNCTION),
                 errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist",
                        pg_encoding_to_char(src_encoding),
                        pg_encoding_to_char(dest_encoding))));
 
    /*
     * Allocate space for conversion result, being wary of integer overflow.
     *
     * len * MAX_CONVERSION_GROWTH is typically a vast overestimate of the
     * required space, so it might exceed MaxAllocSize even though the result
     * would actually fit.  We do not want to hand back a result string that
     * exceeds MaxAllocSize, because callers might not cope gracefully --- but
     * if we just allocate more than that, and don't use it, that's fine.
     */
    if ((Size) len >= (MaxAllocHugeSize / (Size) MAX_CONVERSION_GROWTH))
        ereport(ERROR,
                (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
                 errmsg("out of memory"),
                 errdetail("String of %d bytes is too long for encoding conversion.",
                           len)));
 
    result = (unsigned char *)
        MemoryContextAllocHuge(CurrentMemoryContext,
                               (Size) len * MAX_CONVERSION_GROWTH + 1);
 
    (void) OidFunctionCall6(proc,
                            Int32GetDatum(src_encoding),
                            Int32GetDatum(dest_encoding),
                            CStringGetDatum((char *) src),
                            CStringGetDatum((char *) result),
                            Int32GetDatum(len),
                            BoolGetDatum(false));
 
    /*
     * If the result is large, it's worth repalloc'ing to release any extra
     * space we asked for.  The cutoff here is somewhat arbitrary, but we
     * *must* check when len * MAX_CONVERSION_GROWTH exceeds MaxAllocSize.
     */
    if (len > 1000000)
    {
        Size        resultlen = strlen((char *) result);
 
        if (resultlen >= MaxAllocSize)
            ereport(ERROR,
                    (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
                     errmsg("out of memory"),
                     errdetail("String of %d bytes is too long for encoding conversion.",
                               len)));
 
        result = (unsigned char *) repalloc(result, resultlen + 1);
    }
 
    return result;
}

References BoolGetDatum(), CStringGetDatum(), CurrentMemoryContext, elog, ereport, errcode(), errdetail(), errmsg(), ERROR, FindDefaultConversionProc(), Int32GetDatum(), IsTransactionState(), len, MAX_CONVERSION_GROWTH, MaxAllocHugeSize, MaxAllocSize, MemoryContextAllocHuge(), OidFunctionCall6, OidIsValid, pg_encoding_to_char, PG_SQL_ASCII, pg_verify_mbstr(), and repalloc().

Referenced by convert_charset(), pg_any_to_server(), pg_convert(), and pg_server_to_any().

pg_do_encoding_conversion_buf()

int pg_do_encoding_conversion_buf	(	Oid	proc,
		int	src_encoding,
		int	dest_encoding,
		unsigned char *	src,
		int	srclen,
		unsigned char *	dest,
		int	destlen,
		bool	noError
	)

Definition at line 470 of file mbutils.c.

{
    Datum       result;
 
    /*
     * If the destination buffer is not large enough to hold the result in the
     * worst case, limit the input size passed to the conversion function.
     */
    if ((Size) srclen >= ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH))
        srclen = ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH);
 
    result = OidFunctionCall6(proc,
                              Int32GetDatum(src_encoding),
                              Int32GetDatum(dest_encoding),
                              CStringGetDatum((char *) src),
                              CStringGetDatum((char *) dest),
                              Int32GetDatum(srclen),
                              BoolGetDatum(noError));
    return DatumGetInt32(result);
}

References BoolGetDatum(), CStringGetDatum(), DatumGetInt32(), generate_unaccent_rules::dest, Int32GetDatum(), MAX_CONVERSION_GROWTH, and OidFunctionCall6.

Referenced by CopyConversionError(), CopyConvertBuf(), and test_enc_conversion().

pg_dsplen()

int pg_dsplen

(

const char *

mbstr

)

Definition at line 1031 of file mbutils.c.

{
    return pg_wchar_table[DatabaseEncoding->encoding].dsplen((const unsigned char *) mbstr);
}

References DatabaseEncoding, pg_wchar_tbl::dsplen, pg_enc2name::encoding, and pg_wchar_table.

Referenced by p_isspecial().

pg_encoding_max_length_sql()

Datum pg_encoding_max_length_sql

(

PG_FUNCTION_ARGS

)

Definition at line 645 of file mbutils.c.

{
    int         encoding = PG_GETARG_INT32(0);
 
    if (PG_VALID_ENCODING(encoding))
        PG_RETURN_INT32(pg_wchar_table[encoding].maxmblen);
    else
        PG_RETURN_NULL();
}

References encoding, PG_GETARG_INT32, PG_RETURN_INT32, PG_RETURN_NULL, PG_VALID_ENCODING, and pg_wchar_table.

pg_encoding_mb2wchar_with_len()

int pg_encoding_mb2wchar_with_len	(	int	encoding,
		const char *	from,
		pg_wchar *	to,
		int	len
	)

Definition at line 994 of file mbutils.c.

{
    return pg_wchar_table[encoding].mb2wchar_with_len((const unsigned char *) from, to, len);
}

References encoding, len, pg_wchar_tbl::mb2wchar_with_len, and pg_wchar_table.

pg_encoding_mbcliplen()

int pg_encoding_mbcliplen	(	int	encoding,
		const char *	mbstr,
		int	len,
		int	limit
	)

Definition at line 1094 of file mbutils.c.

{
    mblen_converter mblen_fn;
    int         clen = 0;
    int         l;
 
    /* optimization for single byte encoding */
    if (pg_encoding_max_length(encoding) == 1)
        return cliplen(mbstr, len, limit);
 
    mblen_fn = pg_wchar_table[encoding].mblen;
 
    while (len > 0 && *mbstr)
    {
        l = (*mblen_fn) ((const unsigned char *) mbstr);
        if ((clen + l) > limit)
            break;
        clen += l;
        if (clen == limit)
            break;
        len -= l;
        mbstr += l;
    }
    return clen;
}

References cliplen(), encoding, len, pg_wchar_tbl::mblen, pg_encoding_max_length(), and pg_wchar_table.

Referenced by pg_mbcliplen().

PG_encoding_to_char()

Datum PG_encoding_to_char

(

PG_FUNCTION_ARGS

)

Definition at line 1294 of file mbutils.c.

{
    int32       encoding = PG_GETARG_INT32(0);
    const char *encoding_name = pg_encoding_to_char(encoding);
 
    return DirectFunctionCall1(namein, CStringGetDatum(encoding_name));
}

References CStringGetDatum(), DirectFunctionCall1, encoding, namein(), pg_encoding_to_char, and PG_GETARG_INT32.

pg_encoding_wchar2mb_with_len()

int pg_encoding_wchar2mb_with_len	(	int	encoding,
		const pg_wchar *	from,
		char *	to,
		int	len
	)

Definition at line 1016 of file mbutils.c.

{
    return pg_wchar_table[encoding].wchar2mb_with_len(from, (unsigned char *) to, len);
}

References encoding, len, pg_wchar_table, and pg_wchar_tbl::wchar2mb_with_len.

pg_eucjp_increment()

static bool pg_eucjp_increment ( unsigned char *  charptr, int  length  )

static

Definition at line 1438 of file mbutils.c.

{
    unsigned char c1,
                c2;
    int         i;
 
    c1 = *charptr;
 
    switch (c1)
    {
        case SS2:               /* JIS X 0201 */
            if (length != 2)
                return false;
 
            c2 = charptr[1];
 
            if (c2 >= 0xdf)
                charptr[0] = charptr[1] = 0xa1;
            else if (c2 < 0xa1)
                charptr[1] = 0xa1;
            else
                charptr[1]++;
            break;
 
        case SS3:               /* JIS X 0212 */
            if (length != 3)
                return false;
 
            for (i = 2; i > 0; i--)
            {
                c2 = charptr[i];
                if (c2 < 0xa1)
                {
                    charptr[i] = 0xa1;
                    return true;
                }
                else if (c2 < 0xfe)
                {
                    charptr[i]++;
                    return true;
                }
            }
 
            /* Out of 3-byte code region */
            return false;
 
        default:
            if (IS_HIGHBIT_SET(c1)) /* JIS X 0208? */
            {
                if (length != 2)
                    return false;
 
                for (i = 1; i >= 0; i--)
                {
                    c2 = charptr[i];
                    if (c2 < 0xa1)
                    {
                        charptr[i] = 0xa1;
                        return true;
                    }
                    else if (c2 < 0xfe)
                    {
                        charptr[i]++;
                        return true;
                    }
                }
 
                /* Out of 2 byte code region */
                return false;
            }
            else
            {                   /* ASCII, single byte */
                if (c1 > 0x7e)
                    return false;
                (*charptr)++;
            }
            break;
    }
 
    return true;
}

References i, IS_HIGHBIT_SET, SS2, and SS3.

Referenced by pg_database_encoding_character_incrementer().

pg_generic_charinc()

static bool pg_generic_charinc ( unsigned char *  charptr, int  len  )

static

Definition at line 1326 of file mbutils.c.

{
    unsigned char *lastbyte = charptr + len - 1;
    mbchar_verifier mbverify;
 
    /* We can just invoke the character verifier directly. */
    mbverify = pg_wchar_table[GetDatabaseEncoding()].mbverifychar;
 
    while (*lastbyte < (unsigned char) 255)
    {
        (*lastbyte)++;
        if ((*mbverify) (charptr, len) == len)
            return true;
    }
 
    return false;
}

References GetDatabaseEncoding(), len, pg_wchar_tbl::mbverifychar, and pg_wchar_table.

Referenced by pg_database_encoding_character_incrementer().

pg_get_client_encoding()

int pg_get_client_encoding

(

void

)

Definition at line 337 of file mbutils.c.

{
    return ClientEncoding->encoding;
}

References ClientEncoding, and pg_enc2name::encoding.

Referenced by BeginCopyFrom(), BeginCopyTo(), and xml_send().

pg_get_client_encoding_name()

const char * pg_get_client_encoding_name

(

void

)

Definition at line 346 of file mbutils.c.

{
    return ClientEncoding->name;
}

References ClientEncoding, and pg_enc2name::name.

pg_mb2wchar()

int pg_mb2wchar	(	const char *	from,
		pg_wchar *	to
	)

Definition at line 980 of file mbutils.c.

{
    return pg_wchar_table[DatabaseEncoding->encoding].mb2wchar_with_len((const unsigned char *) from, to, strlen(from));
}

References DatabaseEncoding, pg_enc2name::encoding, pg_wchar_tbl::mb2wchar_with_len, and pg_wchar_table.

pg_mb2wchar_with_len()

int pg_mb2wchar_with_len	(	const char *	from,
		pg_wchar *	to,
		int	len
	)

Definition at line 987 of file mbutils.c.

{
    return pg_wchar_table[DatabaseEncoding->encoding].mb2wchar_with_len((const unsigned char *) from, to, len);
}

References DatabaseEncoding, pg_enc2name::encoding, len, pg_wchar_tbl::mb2wchar_with_len, and pg_wchar_table.

Referenced by CheckAffix(), NIAddAffix(), RE_compile(), RE_compile_and_cache(), RE_execute(), regcomp_auth_token(), regexec_auth_token(), replace_text_regexp(), setup_regexp_matches(), setup_test_matches(), test_re_compile(), and TParserInit().

pg_mbcharcliplen()

int pg_mbcharcliplen	(	const char *	mbstr,
		int	len,
		int	limit
	)

Definition at line 1126 of file mbutils.c.

{
    int         clen = 0;
    int         nch = 0;
    int         l;
 
    /* optimization for single byte encoding */
    if (pg_database_encoding_max_length() == 1)
        return cliplen(mbstr, len, limit);
 
    while (len > 0 && *mbstr)
    {
        l = pg_mblen(mbstr);
        nch++;
        if (nch > limit)
            break;
        clen += l;
        len -= l;
        mbstr += l;
    }
    return clen;
}

References cliplen(), len, pg_database_encoding_max_length(), and pg_mblen().

Referenced by bpchar(), bpchar_input(), text_left(), text_right(), varchar(), and varchar_input().

pg_mbcliplen()

int pg_mbcliplen	(	const char *	mbstr,
		int	len,
		int	limit
	)

Definition at line 1084 of file mbutils.c.

{
    return pg_encoding_mbcliplen(DatabaseEncoding->encoding, mbstr,
                                 len, limit);
}

References DatabaseEncoding, pg_enc2name::encoding, len, and pg_encoding_mbcliplen().

Referenced by appendStringInfoStringQuoted(), bpchar_name(), ChooseIndexColumnNames(), CopyLimitPrintoutLength(), ExecBuildSlotPartitionKeyDescription(), ExecBuildSlotValueDescription(), make_colname_unique(), make_greater_string(), makeMultirangeTypeName(), makeObjectName(), MemoryContextStatsPrint(), nameconcatoid(), namein(), pgstat_clip_activity(), pgstat_report_appname(), PutMemoryContextsStatsTupleStore(), set_rtable_names(), text_name(), text_to_cstring_buffer(), and truncate_identifier().

pg_mblen()

int pg_mblen

(

const char *

mbstr

)

Definition at line 1024 of file mbutils.c.

{
    return pg_wchar_table[DatabaseEncoding->encoding].mblen((const unsigned char *) mbstr);
}

References DatabaseEncoding, pg_enc2name::encoding, pg_wchar_tbl::mblen, and pg_wchar_table.

Referenced by addCompoundAffixFlagValue(), bit_in(), charlen_to_bytelen(), DCH_from_char(), dotrim(), find_word(), findchar(), findchar2(), findwrd(), gbt_var_node_cp_len(), get_modifiers(), get_nextfield(), get_wildcard_part(), getlexeme(), getNextFlagFromString(), gettoken_query(), gettoken_query_standard(), gettoken_query_websearch(), gettoken_tsvector(), hex_decode_safe(), infix(), initTrie(), lpad(), make_trigrams(), map_sql_identifier_to_xml_name(), map_xml_name_to_sql_identifier(), match_prosrc_to_literal(), mb_strchr(), NIImportAffixes(), NIImportDictionary(), NIImportOOAffixes(), NUM_eat_non_data_chars(), NUM_processor(), parse_affentry(), parse_format(), parse_lquery(), parse_ltree(), parse_or_operator(), parse_re_flags(), parse_test_flags(), pg_base64_decode(), pg_mbcharcliplen(), pg_mbstrlen(), pg_mbstrlen_with_len(), prssyntaxerror(), px_crypt_shacrypt(), readstoplist(), report_json_context(), rpad(), RS_compile(), RS_execute(), RS_isRegis(), similar_escape_internal(), split_text(), t_isalnum(), t_isalpha(), text_format(), text_position_next(), text_position_next_internal(), text_reverse(), text_substring(), text_to_bits(), textregexreplace(), thesaurusRead(), TParserGet(), translate(), ts_stat_sql(), tsvectorout(), unaccent_lexize(), varbit_in(), varstr_levenshtein(), and wchareq().

pg_mbstrlen()

int pg_mbstrlen

(

const char *

mbstr

)

Definition at line 1038 of file mbutils.c.

{
    int         len = 0;
 
    /* optimization for single byte encoding */
    if (pg_database_encoding_max_length() == 1)
        return strlen(mbstr);
 
    while (*mbstr)
    {
        mbstr += pg_mblen(mbstr);
        len++;
    }
    return len;
}

References len, pg_database_encoding_max_length(), and pg_mblen().

Referenced by NUM_processor(), and text_format_append_string().

pg_mbstrlen_with_len()

int pg_mbstrlen_with_len	(	const char *	mbstr,
		int	limit
	)

Definition at line 1058 of file mbutils.c.

{
    int         len = 0;
 
    /* optimization for single byte encoding */
    if (pg_database_encoding_max_length() == 1)
        return limit;
 
    while (limit > 0 && *mbstr)
    {
        int         l = pg_mblen(mbstr);
 
        limit -= l;
        mbstr += l;
        len++;
    }
    return len;
}

References len, pg_database_encoding_max_length(), and pg_mblen().

Referenced by bpchar(), bpchar_input(), bpcharlen(), executor_errposition(), lpad(), match_prosrc_to_query(), parser_errposition(), plpgsql_scanner_errposition(), rpad(), scanner_errposition(), similar_escape_internal(), text_left(), text_length(), text_position_get_match_pos(), text_right(), text_substring(), unicode_assigned(), unicode_is_normalized(), unicode_normalize_func(), and varstr_levenshtein().

pg_server_to_any()

char * pg_server_to_any	(	const char *	s,
		int	len,
		int	encoding
	)

Definition at line 750 of file mbutils.c.

{
    if (len <= 0)
        return unconstify(char *, s);   /* empty string is always valid */
 
    if (encoding == DatabaseEncoding->encoding ||
        encoding == PG_SQL_ASCII)
        return unconstify(char *, s);   /* assume data is valid */
 
    if (DatabaseEncoding->encoding == PG_SQL_ASCII)
    {
        /* No conversion is possible, but we must validate the result */
        (void) pg_verify_mbstr(encoding, s, len, false);
        return unconstify(char *, s);
    }
 
    /* Fast path if we can use cached conversion function */
    if (encoding == ClientEncoding->encoding)
        return perform_default_encoding_conversion(s, len, false);
 
    /* General case ... will not work outside transactions */
    return (char *) pg_do_encoding_conversion((unsigned char *) unconstify(char *, s),
                                              len,
                                              DatabaseEncoding->encoding,
                                              encoding);
}

References ClientEncoding, DatabaseEncoding, encoding, pg_enc2name::encoding, len, perform_default_encoding_conversion(), pg_do_encoding_conversion(), PG_SQL_ASCII, pg_verify_mbstr(), and unconstify.

Referenced by compareStrings(), CopyAttributeOutCSV(), CopyAttributeOutText(), CopyToTextLikeStart(), daitch_mokotoff(), dsnowball_lexize(), hv_fetch_string(), hv_store_string(), pg_server_to_client(), PLyUnicode_FromStringAndSize(), and utf_e2u().

pg_server_to_client()

char * pg_server_to_client	(	const char *	s,
		int	len
	)

Definition at line 739 of file mbutils.c.

{
    return pg_server_to_any(s, len, ClientEncoding->encoding);
}

References ClientEncoding, pg_enc2name::encoding, len, and pg_server_to_any().

Referenced by pq_puttextmessage(), pq_sendcountedtext(), pq_sendstring(), pq_sendtext(), and pq_writestring().

pg_unicode_to_server()

void pg_unicode_to_server	(	pg_wchar	c,
		unsigned char *	s
	)

Definition at line 865 of file mbutils.c.

{
    unsigned char c_as_utf8[MAX_MULTIBYTE_CHAR_LEN + 1];
    int         c_as_utf8_len;
    int         server_encoding;
 
    /*
     * Complain if invalid Unicode code point.  The choice of errcode here is
     * debatable, but really our caller should have checked this anyway.
     */
    if (!is_valid_unicode_codepoint(c))
        ereport(ERROR,
                (errcode(ERRCODE_SYNTAX_ERROR),
                 errmsg("invalid Unicode code point")));
 
    /* Otherwise, if it's in ASCII range, conversion is trivial */
    if (c <= 0x7F)
    {
        s[0] = (unsigned char) c;
        s[1] = '\0';
        return;
    }
 
    /* If the server encoding is UTF-8, we just need to reformat the code */
    server_encoding = GetDatabaseEncoding();
    if (server_encoding == PG_UTF8)
    {
        unicode_to_utf8(c, s);
        s[pg_utf_mblen(s)] = '\0';
        return;
    }
 
    /* For all other cases, we must have a conversion function available */
    if (Utf8ToServerConvProc == NULL)
        ereport(ERROR,
                (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
                 errmsg("conversion between %s and %s is not supported",
                        pg_enc2name_tbl[PG_UTF8].name,
                        GetDatabaseEncodingName())));
 
    /* Construct UTF-8 source string */
    unicode_to_utf8(c, c_as_utf8);
    c_as_utf8_len = pg_utf_mblen(c_as_utf8);
    c_as_utf8[c_as_utf8_len] = '\0';
 
    /* Convert, or throw error if we can't */
    FunctionCall6(Utf8ToServerConvProc,
                  Int32GetDatum(PG_UTF8),
                  Int32GetDatum(server_encoding),
                  CStringGetDatum((char *) c_as_utf8),
                  CStringGetDatum((char *) s),
                  Int32GetDatum(c_as_utf8_len),
                  BoolGetDatum(false));
}

References BoolGetDatum(), CStringGetDatum(), ereport, errcode(), errmsg(), ERROR, FunctionCall6, GetDatabaseEncoding(), GetDatabaseEncodingName(), Int32GetDatum(), is_valid_unicode_codepoint(), MAX_MULTIBYTE_CHAR_LEN, name, pg_enc2name_tbl, PG_UTF8, pg_utf_mblen, unicode_to_utf8(), and Utf8ToServerConvProc.

Referenced by addunicode(), addUnicodeChar(), map_xml_name_to_sql_identifier(), str_udeescape(), and unistr().

pg_unicode_to_server_noerror()

bool pg_unicode_to_server_noerror	(	pg_wchar	c,
		unsigned char *	s
	)

Definition at line 927 of file mbutils.c.

{
    unsigned char c_as_utf8[MAX_MULTIBYTE_CHAR_LEN + 1];
    int         c_as_utf8_len;
    int         converted_len;
    int         server_encoding;
 
    /* Fail if invalid Unicode code point */
    if (!is_valid_unicode_codepoint(c))
        return false;
 
    /* Otherwise, if it's in ASCII range, conversion is trivial */
    if (c <= 0x7F)
    {
        s[0] = (unsigned char) c;
        s[1] = '\0';
        return true;
    }
 
    /* If the server encoding is UTF-8, we just need to reformat the code */
    server_encoding = GetDatabaseEncoding();
    if (server_encoding == PG_UTF8)
    {
        unicode_to_utf8(c, s);
        s[pg_utf_mblen(s)] = '\0';
        return true;
    }
 
    /* For all other cases, we must have a conversion function available */
    if (Utf8ToServerConvProc == NULL)
        return false;
 
    /* Construct UTF-8 source string */
    unicode_to_utf8(c, c_as_utf8);
    c_as_utf8_len = pg_utf_mblen(c_as_utf8);
    c_as_utf8[c_as_utf8_len] = '\0';
 
    /* Convert, but without throwing error if we can't */
    converted_len = DatumGetInt32(FunctionCall6(Utf8ToServerConvProc,
                                                Int32GetDatum(PG_UTF8),
                                                Int32GetDatum(server_encoding),
                                                CStringGetDatum((char *) c_as_utf8),
                                                CStringGetDatum((char *) s),
                                                Int32GetDatum(c_as_utf8_len),
                                                BoolGetDatum(true)));
 
    /* Conversion was successful iff it consumed the whole input */
    return (converted_len == c_as_utf8_len);
}

References BoolGetDatum(), CStringGetDatum(), DatumGetInt32(), FunctionCall6, GetDatabaseEncoding(), Int32GetDatum(), is_valid_unicode_codepoint(), MAX_MULTIBYTE_CHAR_LEN, PG_UTF8, pg_utf_mblen, unicode_to_utf8(), and Utf8ToServerConvProc.

Referenced by addUnicodeChar(), and json_lex_string().

pg_utf8_increment()

static bool pg_utf8_increment ( unsigned char *  charptr, int  length  )

static

Definition at line 1360 of file mbutils.c.

{
    unsigned char a;
    unsigned char limit;
 
    switch (length)
    {
        default:
            /* reject lengths 5 and 6 for now */
            return false;
        case 4:
            a = charptr[3];
            if (a < 0xBF)
            {
                charptr[3]++;
                break;
            }
            /* FALL THRU */
        case 3:
            a = charptr[2];
            if (a < 0xBF)
            {
                charptr[2]++;
                break;
            }
            /* FALL THRU */
        case 2:
            a = charptr[1];
            switch (*charptr)
            {
                case 0xED:
                    limit = 0x9F;
                    break;
                case 0xF4:
                    limit = 0x8F;
                    break;
                default:
                    limit = 0xBF;
                    break;
            }
            if (a < limit)
            {
                charptr[1]++;
                break;
            }
            /* FALL THRU */
        case 1:
            a = *charptr;
            if (a == 0x7F || a == 0xDF || a == 0xEF || a == 0xF4)
                return false;
            charptr[0]++;
            break;
    }
 
    return true;
}

References a.

Referenced by pg_database_encoding_character_incrementer().

pg_verify_mbstr()

bool pg_verify_mbstr	(	int	encoding,
		const char *	mbstr,
		int	len,
		bool	noError
	)

Definition at line 1567 of file mbutils.c.

{
    int         oklen;
 
    Assert(PG_VALID_ENCODING(encoding));
 
    oklen = pg_wchar_table[encoding].mbverifystr((const unsigned char *) mbstr, len);
    if (oklen != len)
    {
        if (noError)
            return false;
        report_invalid_encoding(encoding, mbstr + oklen, len - oklen);
    }
    return true;
}

References Assert(), encoding, len, pg_wchar_tbl::mbverifystr, PG_VALID_ENCODING, pg_wchar_table, and report_invalid_encoding().

Referenced by AddFileToBackupManifest(), LogicalOutputWrite(), pg_any_to_server(), pg_convert(), pg_do_encoding_conversion(), pg_server_to_any(), pg_verifymbstr(), and read_extension_script_file().

pg_verify_mbstr_len()

int pg_verify_mbstr_len	(	int	encoding,
		const char *	mbstr,
		int	len,
		bool	noError
	)

Definition at line 1598 of file mbutils.c.

{
    mbchar_verifier mbverifychar;
    int         mb_len;
 
    Assert(PG_VALID_ENCODING(encoding));
 
    /*
     * In single-byte encodings, we need only reject nulls (\0).
     */
    if (pg_encoding_max_length(encoding) <= 1)
    {
        const char *nullpos = memchr(mbstr, 0, len);
 
        if (nullpos == NULL)
            return len;
        if (noError)
            return -1;
        report_invalid_encoding(encoding, nullpos, 1);
    }
 
    /* fetch function pointer just once */
    mbverifychar = pg_wchar_table[encoding].mbverifychar;
 
    mb_len = 0;
 
    while (len > 0)
    {
        int         l;
 
        /* fast path for ASCII-subset characters */
        if (!IS_HIGHBIT_SET(*mbstr))
        {
            if (*mbstr != '\0')
            {
                mb_len++;
                mbstr++;
                len--;
                continue;
            }
            if (noError)
                return -1;
            report_invalid_encoding(encoding, mbstr, len);
        }
 
        l = (*mbverifychar) ((const unsigned char *) mbstr, len);
 
        if (l < 0)
        {
            if (noError)
                return -1;
            report_invalid_encoding(encoding, mbstr, len);
        }
 
        mbstr += l;
        len -= l;
        mb_len++;
    }
    return mb_len;
}

References Assert(), encoding, IS_HIGHBIT_SET, len, pg_wchar_tbl::mbverifychar, pg_encoding_max_length(), PG_VALID_ENCODING, pg_wchar_table, and report_invalid_encoding().

Referenced by length_in_encoding().

pg_verifymbstr()

bool pg_verifymbstr	(	const char *	mbstr,
		int	len,
		bool	noError
	)

Definition at line 1557 of file mbutils.c.

{
    return pg_verify_mbstr(GetDatabaseEncoding(), mbstr, len, noError);
}

References GetDatabaseEncoding(), len, and pg_verify_mbstr().

Referenced by char2wchar(), CopyReadAttributesText(), plperl_spi_exec(), plperl_spi_prepare(), plperl_spi_query(), PLy_cursor_query(), PLy_output(), PLy_spi_execute_query(), PLy_spi_prepare(), PLyObject_AsString(), read_text_file(), and spg_text_leaf_consistent().

pg_wchar2mb()

int pg_wchar2mb	(	const pg_wchar *	from,
		char *	to
	)

Definition at line 1002 of file mbutils.c.

{
    return pg_wchar_table[DatabaseEncoding->encoding].wchar2mb_with_len(from, (unsigned char *) to, pg_wchar_strlen(from));
}

References DatabaseEncoding, pg_enc2name::encoding, pg_wchar_strlen(), pg_wchar_table, and pg_wchar_tbl::wchar2mb_with_len.

pg_wchar2mb_with_len()

int pg_wchar2mb_with_len	(	const pg_wchar *	from,
		char *	to,
		int	len
	)

Definition at line 1009 of file mbutils.c.

{
    return pg_wchar_table[DatabaseEncoding->encoding].wchar2mb_with_len(from, (unsigned char *) to, len);
}

References DatabaseEncoding, pg_enc2name::encoding, len, pg_wchar_table, and pg_wchar_tbl::wchar2mb_with_len.

Referenced by build_regexp_match_result(), build_regexp_split_result(), build_test_match_result(), convertPgWchar(), and regexp_fixed_prefix().

PrepareClientEncoding()

int PrepareClientEncoding

(

int

encoding

)

Definition at line 111 of file mbutils.c.

{
    int         current_server_encoding;
    ListCell   *lc;
 
    if (!PG_VALID_FE_ENCODING(encoding))
        return -1;
 
    /* Can't do anything during startup, per notes above */
    if (!backend_startup_complete)
        return 0;
 
    current_server_encoding = GetDatabaseEncoding();
 
    /*
     * Check for cases that require no conversion function.
     */
    if (current_server_encoding == encoding ||
        current_server_encoding == PG_SQL_ASCII ||
        encoding == PG_SQL_ASCII)
        return 0;
 
    if (IsTransactionState())
    {
        /*
         * If we're in a live transaction, it's safe to access the catalogs,
         * so look up the functions.  We repeat the lookup even if the info is
         * already cached, so that we can react to changes in the contents of
         * pg_conversion.
         */
        Oid         to_server_proc,
                    to_client_proc;
        ConvProcInfo *convinfo;
        MemoryContext oldcontext;
 
        to_server_proc = FindDefaultConversionProc(encoding,
                                                   current_server_encoding);
        if (!OidIsValid(to_server_proc))
            return -1;
        to_client_proc = FindDefaultConversionProc(current_server_encoding,
                                                   encoding);
        if (!OidIsValid(to_client_proc))
            return -1;
 
        /*
         * Load the fmgr info into TopMemoryContext (could still fail here)
         */
        convinfo = (ConvProcInfo *) MemoryContextAlloc(TopMemoryContext,
                                                       sizeof(ConvProcInfo));
        convinfo->s_encoding = current_server_encoding;
        convinfo->c_encoding = encoding;
        fmgr_info_cxt(to_server_proc, &convinfo->to_server_info,
                      TopMemoryContext);
        fmgr_info_cxt(to_client_proc, &convinfo->to_client_info,
                      TopMemoryContext);
 
        /* Attach new info to head of list */
        oldcontext = MemoryContextSwitchTo(TopMemoryContext);
        ConvProcList = lcons(convinfo, ConvProcList);
        MemoryContextSwitchTo(oldcontext);
 
        /*
         * We cannot yet remove any older entry for the same encoding pair,
         * since it could still be in use.  SetClientEncoding will clean up.
         */
 
        return 0;               /* success */
    }
    else
    {
        /*
         * If we're not in a live transaction, the only thing we can do is
         * restore a previous setting using the cache.  This covers all
         * transaction-rollback cases.  The only case it might not work for is
         * trying to change client_encoding on the fly by editing
         * postgresql.conf and SIGHUP'ing.  Which would probably be a stupid
         * thing to do anyway.
         */
        foreach(lc, ConvProcList)
        {
            ConvProcInfo *oldinfo = (ConvProcInfo *) lfirst(lc);
 
            if (oldinfo->s_encoding == current_server_encoding &&
                oldinfo->c_encoding == encoding)
                return 0;
        }
 
        return -1;              /* it's not cached, so fail */
    }
}

References backend_startup_complete, ConvProcInfo::c_encoding, ConvProcList, encoding, FindDefaultConversionProc(), fmgr_info_cxt(), GetDatabaseEncoding(), IsTransactionState(), lcons(), lfirst, MemoryContextAlloc(), MemoryContextSwitchTo(), OidIsValid, PG_SQL_ASCII, PG_VALID_FE_ENCODING, ConvProcInfo::s_encoding, ConvProcInfo::to_client_info, ConvProcInfo::to_server_info, and TopMemoryContext.

Referenced by check_client_encoding(), and InitializeClientEncoding().

report_invalid_encoding()

void report_invalid_encoding	(	int	encoding,
		const char *	mbstr,
		int	len
	)

Definition at line 1699 of file mbutils.c.

{
    int         l = pg_encoding_mblen_or_incomplete(encoding, mbstr, len);
    char        buf[8 * 5 + 1];
    char       *p = buf;
    int         j,
                jlimit;
 
    jlimit = Min(l, len);
    jlimit = Min(jlimit, 8);    /* prevent buffer overrun */
 
    for (j = 0; j < jlimit; j++)
    {
        p += sprintf(p, "0x%02x", (unsigned char) mbstr[j]);
        if (j < jlimit - 1)
            p += sprintf(p, " ");
    }
 
    ereport(ERROR,
            (errcode(ERRCODE_CHARACTER_NOT_IN_REPERTOIRE),
             errmsg("invalid byte sequence for encoding \"%s\": %s",
                    pg_enc2name_tbl[encoding].name,
                    buf)));
}

References buf, encoding, ereport, errcode(), errmsg(), ERROR, j, len, Min, name, pg_enc2name_tbl, pg_encoding_mblen_or_incomplete(), and sprintf.

Referenced by big52euc_tw(), big52mic(), CopyConversionError(), euc_cn2mic(), euc_jis_20042shift_jis_2004(), euc_jp2mic(), euc_jp2sjis(), euc_kr2mic(), euc_tw2big5(), euc_tw2mic(), iso8859_1_to_utf8(), latin2mic(), latin2mic_with_table(), local2local(), LocalToUtf(), mic2big5(), mic2euc_cn(), mic2euc_jp(), mic2euc_kr(), mic2euc_tw(), mic2latin(), mic2latin_with_table(), mic2sjis(), pg_verify_mbstr(), pg_verify_mbstr_len(), shift_jis_20042euc_jis_2004(), sjis2euc_jp(), sjis2mic(), test_enc_conversion(), utf8_to_iso8859_1(), and UtfToLocal().

report_untranslatable_char()

void report_untranslatable_char	(	int	src_encoding,
		int	dest_encoding,
		const char *	mbstr,
		int	len
	)

Definition at line 1731 of file mbutils.c.

{
    int         l;
    char        buf[8 * 5 + 1];
    char       *p = buf;
    int         j,
                jlimit;
 
    /*
     * We probably could use plain pg_encoding_mblen(), because
     * gb18030_to_utf8() verifies before it converts.  All conversions should.
     * For src_encoding!=GB18030, len>0 meets pg_encoding_mblen() needs.  Even
     * so, be defensive, since a buggy conversion might pass invalid data.
     * This is not a performance-critical path.
     */
    l = pg_encoding_mblen_or_incomplete(src_encoding, mbstr, len);
    jlimit = Min(l, len);
    jlimit = Min(jlimit, 8);    /* prevent buffer overrun */
 
    for (j = 0; j < jlimit; j++)
    {
        p += sprintf(p, "0x%02x", (unsigned char) mbstr[j]);
        if (j < jlimit - 1)
            p += sprintf(p, " ");
    }
 
    ereport(ERROR,
            (errcode(ERRCODE_UNTRANSLATABLE_CHARACTER),
             errmsg("character with byte sequence %s in encoding \"%s\" has no equivalent in encoding \"%s\"",
                    buf,
                    pg_enc2name_tbl[src_encoding].name,
                    pg_enc2name_tbl[dest_encoding].name)));
}

References buf, ereport, errcode(), errmsg(), ERROR, j, len, Min, name, pg_enc2name_tbl, pg_encoding_mblen_or_incomplete(), and sprintf.

Referenced by big52euc_tw(), big52mic(), euc_tw2big5(), latin2mic_with_table(), local2local(), LocalToUtf(), mic2big5(), mic2euc_cn(), mic2euc_jp(), mic2euc_kr(), mic2euc_tw(), mic2latin(), mic2latin_with_table(), mic2sjis(), utf8_to_iso8859_1(), and UtfToLocal().

SetClientEncoding()

int SetClientEncoding

(

int

encoding

)

Definition at line 209 of file mbutils.c.

{
    int         current_server_encoding;
    bool        found;
    ListCell   *lc;
 
    if (!PG_VALID_FE_ENCODING(encoding))
        return -1;
 
    /* Can't do anything during startup, per notes above */
    if (!backend_startup_complete)
    {
        pending_client_encoding = encoding;
        return 0;
    }
 
    current_server_encoding = GetDatabaseEncoding();
 
    /*
     * Check for cases that require no conversion function.
     */
    if (current_server_encoding == encoding ||
        current_server_encoding == PG_SQL_ASCII ||
        encoding == PG_SQL_ASCII)
    {
        ClientEncoding = &pg_enc2name_tbl[encoding];
        ToServerConvProc = NULL;
        ToClientConvProc = NULL;
        return 0;
    }
 
    /*
     * Search the cache for the entry previously prepared by
     * PrepareClientEncoding; if there isn't one, we lose.  While at it,
     * release any duplicate entries so that repeated Prepare/Set cycles don't
     * leak memory.
     */
    found = false;
    foreach(lc, ConvProcList)
    {
        ConvProcInfo *convinfo = (ConvProcInfo *) lfirst(lc);
 
        if (convinfo->s_encoding == current_server_encoding &&
            convinfo->c_encoding == encoding)
        {
            if (!found)
            {
                /* Found newest entry, so set up */
                ClientEncoding = &pg_enc2name_tbl[encoding];
                ToServerConvProc = &convinfo->to_server_info;
                ToClientConvProc = &convinfo->to_client_info;
                found = true;
            }
            else
            {
                /* Duplicate entry, release it */
                ConvProcList = foreach_delete_current(ConvProcList, lc);
                pfree(convinfo);
            }
        }
    }
 
    if (found)
        return 0;               /* success */
    else
        return -1;              /* it's not cached, so fail */
}

References backend_startup_complete, ConvProcInfo::c_encoding, ClientEncoding, ConvProcList, encoding, foreach_delete_current, GetDatabaseEncoding(), lfirst, pending_client_encoding, pfree(), pg_enc2name_tbl, PG_SQL_ASCII, PG_VALID_FE_ENCODING, ConvProcInfo::s_encoding, ConvProcInfo::to_client_info, ConvProcInfo::to_server_info, ToClientConvProc, and ToServerConvProc.

Referenced by assign_client_encoding(), InitializeClientEncoding(), and ParallelWorkerMain().

SetDatabaseEncoding()

void SetDatabaseEncoding

(

int

encoding

)

Definition at line 1162 of file mbutils.c.

{
    if (!PG_VALID_BE_ENCODING(encoding))
        elog(ERROR, "invalid database encoding: %d", encoding);
 
    DatabaseEncoding = &pg_enc2name_tbl[encoding];
    Assert(DatabaseEncoding->encoding == encoding);
}

References Assert(), DatabaseEncoding, elog, encoding, pg_enc2name::encoding, ERROR, pg_enc2name_tbl, and PG_VALID_BE_ENCODING.

Referenced by CheckMyDatabase().

SetMessageEncoding()

void SetMessageEncoding

(

int

encoding

)

Definition at line 1172 of file mbutils.c.

{
    /* Some calls happen before we can elog()! */
    Assert(PG_VALID_ENCODING(encoding));
 
    MessageEncoding = &pg_enc2name_tbl[encoding];
    Assert(MessageEncoding->encoding == encoding);
}

References Assert(), encoding, pg_enc2name::encoding, MessageEncoding, pg_enc2name_tbl, and PG_VALID_ENCODING.

Referenced by pg_perm_setlocale().

Variable Documentation

backend_startup_complete

bool backend_startup_complete = false

static

Definition at line 91 of file mbutils.c.

Referenced by InitializeClientEncoding(), PrepareClientEncoding(), and SetClientEncoding().

ClientEncoding

const pg_enc2name* ClientEncoding = &pg_enc2name_tbl[PG_SQL_ASCII]

static

Definition at line 81 of file mbutils.c.

Referenced by perform_default_encoding_conversion(), pg_any_to_server(), pg_client_encoding(), pg_client_to_server(), pg_get_client_encoding(), pg_get_client_encoding_name(), pg_server_to_any(), pg_server_to_client(), and SetClientEncoding().

ConvProcList

List* ConvProcList = NIL

static

Definition at line 62 of file mbutils.c.

Referenced by PrepareClientEncoding(), and SetClientEncoding().

DatabaseEncoding

const pg_enc2name* DatabaseEncoding = &pg_enc2name_tbl[PG_SQL_ASCII]

static

Definition at line 82 of file mbutils.c.

Referenced by getdatabaseencoding(), GetDatabaseEncoding(), GetDatabaseEncodingName(), perform_default_encoding_conversion(), pg_any_to_server(), pg_convert_from(), pg_convert_to(), pg_dsplen(), pg_mb2wchar(), pg_mb2wchar_with_len(), pg_mbcliplen(), pg_mblen(), pg_server_to_any(), pg_wchar2mb(), pg_wchar2mb_with_len(), and SetDatabaseEncoding().

MessageEncoding

const pg_enc2name* MessageEncoding = &pg_enc2name_tbl[PG_SQL_ASCII]

static

Definition at line 83 of file mbutils.c.

Referenced by GetMessageEncoding(), and SetMessageEncoding().

pending_client_encoding

int pending_client_encoding = PG_SQL_ASCII

static

Definition at line 92 of file mbutils.c.

Referenced by InitializeClientEncoding(), and SetClientEncoding().

ToClientConvProc

FmgrInfo* ToClientConvProc = NULL

static

Definition at line 69 of file mbutils.c.

Referenced by perform_default_encoding_conversion(), and SetClientEncoding().

ToServerConvProc

FmgrInfo* ToServerConvProc = NULL

static

Definition at line 68 of file mbutils.c.

Referenced by perform_default_encoding_conversion(), and SetClientEncoding().

Utf8ToServerConvProc

FmgrInfo* Utf8ToServerConvProc = NULL

static

Definition at line 76 of file mbutils.c.

Referenced by InitializeClientEncoding(), pg_unicode_to_server(), and pg_unicode_to_server_noerror().