added code for unac to perform pure case-folding
This commit is contained in:
Jean-Francois Dockes
+23
-9
@@ -341,36 +341,50 @@ sub source {
|
||||
@values = ();
|
||||
}
|
||||
$code_value = uc(sprintf("%04x", $code_value));
|
||||
#print "$code_value UNAC ";
|
||||
if(exists($decomposition->{$code_value})) {
|
||||
push(@values, $decomposition->{$code_value});
|
||||
#print "$decomposition->{$code_value} ";
|
||||
} else {
|
||||
push(@values, "FFFF");
|
||||
#print "FFFF ";
|
||||
}
|
||||
# We also push the case-folded version of the unaccented char
|
||||
# Note that by pushing the case-folded version of the original
|
||||
# char, we'd have the possibility of independant unaccenting and
|
||||
# case folding, but with less performance.
|
||||
# We could also keep the three chunks, using a little more memory
|
||||
# We push both the case-folded version of the unaccented char
|
||||
# and the case-folded version of the original one. This
|
||||
# makes the table a little bigger, but allows
|
||||
# independantly unaccenting, folding or both
|
||||
#print "UNACFOLD ";
|
||||
if(exists($decomposition->{$code_value})) {
|
||||
my($cv);
|
||||
my(@vl);
|
||||
foreach $cv (split(' ', $decomposition->{$code_value})) {
|
||||
if(exists($casefold->{$cv})) {
|
||||
push(@vl, $casefold->{$cv});
|
||||
#print "$casefold->{$cv} ";
|
||||
} else {
|
||||
push(@vl, $cv);
|
||||
#print "$cv ";
|
||||
}
|
||||
}
|
||||
#print STDERR "Pushing " . join(" ", @vl) . " for " .
|
||||
#$code_value . "\n";
|
||||
push(@values, join(" ", @vl));
|
||||
} else {
|
||||
if(exists($casefold->{$code_value})) {
|
||||
push(@values, $casefold->{$code_value});
|
||||
#print "$casefold->{$code_value} ";
|
||||
} else {
|
||||
push(@values, "FFFF");
|
||||
#print "FFFF ";
|
||||
}
|
||||
}
|
||||
#print "FOLD ";
|
||||
if(exists($casefold->{$code_value})) {
|
||||
push(@values, $casefold->{$code_value});
|
||||
#print "$casefold->{$code_value} ";
|
||||
} else {
|
||||
push(@values, "FFFF");
|
||||
#print "FFFF ";
|
||||
}
|
||||
#print "\n";
|
||||
}
|
||||
print STDERR scalar(@blocks) . " blocks of " . $block_count . " entries, factorized $duplicate blocks\n\t" if($verbose);
|
||||
my($block_size) = 0;
|
||||
@@ -466,7 +480,7 @@ EOF
|
||||
$block_number++;
|
||||
}
|
||||
my($position_type) = $highest_position >= 256 ? "short" : "char";
|
||||
my($positions_out) = "unsigned $position_type unac_positions[UNAC_BLOCK_COUNT][2*UNAC_BLOCK_SIZE + 1] = {\n";
|
||||
my($positions_out) = "unsigned $position_type unac_positions[UNAC_BLOCK_COUNT][3*UNAC_BLOCK_SIZE + 1] = {\n";
|
||||
|
||||
$positions_out .= join(",\n", @positions_out);
|
||||
$positions_out .= "\n};\n";
|
||||
@@ -481,7 +495,7 @@ EOF
|
||||
my($declarations);
|
||||
$declarations = <<EOF;
|
||||
extern unsigned short unac_indexes[UNAC_INDEXES_SIZE];
|
||||
extern unsigned $position_type unac_positions[UNAC_BLOCK_COUNT][2*UNAC_BLOCK_SIZE + 1];
|
||||
extern unsigned $position_type unac_positions[UNAC_BLOCK_COUNT][3*UNAC_BLOCK_SIZE + 1];
|
||||
extern unsigned short* unac_data_table[UNAC_BLOCK_COUNT];
|
||||
EOF
|
||||
for($block_number = 0; $block_number < $block_count; $block_number++) {
|
||||
|
||||
+988
-946
File diff suppressed because it is too large
Load Diff
+24
-22
@@ -35,7 +35,7 @@ extern "C" {
|
||||
#define UNAC_BLOCK_SHIFT 4
|
||||
#define UNAC_BLOCK_MASK ((1 << UNAC_BLOCK_SHIFT) - 1)
|
||||
#define UNAC_BLOCK_SIZE (1 << UNAC_BLOCK_SHIFT)
|
||||
#define UNAC_BLOCK_COUNT 418
|
||||
#define UNAC_BLOCK_COUNT 422
|
||||
#define UNAC_INDEXES_SIZE (0x10000 >> UNAC_BLOCK_SHIFT)
|
||||
/* Generated by builder. Do not modify. End defines */
|
||||
|
||||
@@ -45,38 +45,31 @@ extern "C" {
|
||||
* by <p> is returned in the <l> argument.
|
||||
* The C++ prototype of this macro would be:
|
||||
*
|
||||
* void unac_char(const unsigned short c, unsigned short*& p, int& l)
|
||||
* void unac_char(const unsigned short c, unsigned short*& p, int& l, int o)
|
||||
*
|
||||
* See unac(3) in IMPLEMENTATION NOTES for more information about the
|
||||
* tables (unac_data_table, unac_positions) layout.
|
||||
*
|
||||
* Each transformed char has 3 possible outputs: unaccented, unaccented and
|
||||
* folded, or just folded. These are kept at offset 0,1,2 in the position table
|
||||
*/
|
||||
#define unac_char_utf16(c,p,l) \
|
||||
{ \
|
||||
unsigned short index = unac_indexes[(c) >> UNAC_BLOCK_SHIFT]; \
|
||||
unsigned char position = 2*((c) & UNAC_BLOCK_MASK); \
|
||||
(p) = &(unac_data_table[index][unac_positions[index][position]]); \
|
||||
(l) = unac_positions[index][position + 1] - unac_positions[index][position]; \
|
||||
if((l) == 1 && *(p) == 0xFFFF) { \
|
||||
(p) = 0; \
|
||||
(l) = 0; \
|
||||
} \
|
||||
}
|
||||
|
||||
/*
|
||||
* Save as unac_ but case-folded
|
||||
*/
|
||||
#define unacfold_char_utf16(c,p,l) \
|
||||
#define unac_uf_char_utf16_(c,p,l,o) \
|
||||
{ \
|
||||
unsigned short index = unac_indexes[(c) >> UNAC_BLOCK_SHIFT]; \
|
||||
unsigned char position = 2*((c) & UNAC_BLOCK_MASK)+1; \
|
||||
unsigned short index = unac_indexes[(c) >> UNAC_BLOCK_SHIFT]; \
|
||||
unsigned char position = 3*((c) & UNAC_BLOCK_MASK) + (o); \
|
||||
(p) = &(unac_data_table[index][unac_positions[index][position]]); \
|
||||
(l) = unac_positions[index][position + 1] - unac_positions[index][position]; \
|
||||
(l) = unac_positions[index][position + 1] \
|
||||
- unac_positions[index][position]; \
|
||||
if((l) == 1 && *(p) == 0xFFFF) { \
|
||||
(p) = 0; \
|
||||
(l) = 0; \
|
||||
} \
|
||||
}
|
||||
|
||||
#define unac_char_utf16(c,p,l) unac_uf_char_utf16_((c),(p),(l),0)
|
||||
#define unacfold_char_utf16(c,p,l) unac_uf_char_utf16_((c),(p),(l),1)
|
||||
#define fold_char_utf16(c,p,l) unac_uf_char_utf16_((c),(p),(l),2)
|
||||
|
||||
/*
|
||||
* Return the unaccented equivalent of the UTF-16 string <in> of
|
||||
* length <in_length> in the pointer <out>. The length of the UTF-16
|
||||
@@ -94,6 +87,8 @@ int unac_string_utf16(const char* in, size_t in_length,
|
||||
char** out, size_t* out_length);
|
||||
int unacfold_string_utf16(const char* in, size_t in_length,
|
||||
char** out, size_t* out_length);
|
||||
int fold_string_utf16(const char* in, size_t in_length,
|
||||
char** out, size_t* out_length);
|
||||
|
||||
/*
|
||||
* The semantic of this function is stricly equal to the function
|
||||
@@ -112,6 +107,9 @@ int unac_string(const char* charset,
|
||||
int unacfold_string(const char* charset,
|
||||
const char* in, size_t in_length,
|
||||
char** out, size_t* out_length);
|
||||
int fold_string(const char* charset,
|
||||
const char* in, size_t in_length,
|
||||
char** out, size_t* out_length);
|
||||
|
||||
/* To be called before starting threads in mt programs */
|
||||
void unac_init_mt();
|
||||
@@ -180,7 +178,7 @@ void unac_debug_callback(int level, unac_debug_print_t function, void* data);
|
||||
|
||||
/* Generated by builder. Do not modify. Start declarations */
|
||||
extern unsigned short unac_indexes[UNAC_INDEXES_SIZE];
|
||||
extern unsigned char unac_positions[UNAC_BLOCK_COUNT][2*UNAC_BLOCK_SIZE + 1];
|
||||
extern unsigned char unac_positions[UNAC_BLOCK_COUNT][3*UNAC_BLOCK_SIZE + 1];
|
||||
extern unsigned short* unac_data_table[UNAC_BLOCK_COUNT];
|
||||
extern unsigned short unac_data0[];
|
||||
extern unsigned short unac_data1[];
|
||||
@@ -600,6 +598,10 @@ extern unsigned short unac_data414[];
|
||||
extern unsigned short unac_data415[];
|
||||
extern unsigned short unac_data416[];
|
||||
extern unsigned short unac_data417[];
|
||||
extern unsigned short unac_data418[];
|
||||
extern unsigned short unac_data419[];
|
||||
extern unsigned short unac_data420[];
|
||||
extern unsigned short unac_data421[];
|
||||
/* Generated by builder. Do not modify. End declarations */
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
||||
Reference in New Issue
Block a user