tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

utf8iter driver: read from stdin

Browse Source
This commit is contained in:
Jean-Francois Dockes 2019-07-05 11:33:29 +02:00
parent 75759ed481
commit 3f33d1d0ea
1 changed files with 87 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

166
src/testmains/trutf8iter.cpp
View File

@ -44,17 +44,19 @@ void tryempty()
const char *thisprog; const char *thisprog;
static char usage [] = static char usage [] =
"utf8iter [opts] infile outfile\n" "utf8iter [opts] infile outfile\n"
" converts infile to 32 bits unicode (processor order), for testing\n" " converts infile to 32 bits unicode (processor order), for testing\n"
"-v : print stuff as we go\n" "-v : print stuff as we go\n"
; ;
void Usage() { void Usage() {
fprintf(stderr, "%s:%s\n", thisprog, usage); fprintf(stderr, "%s:%s\n", thisprog, usage);
exit(1); exit(1);
} }
static int op_flags; static int op_flags;
#define OPT_v 0x2 #define OPT_v 0x2
FILE *infout = stdout;
int main(int argc, char **argv) int main(int argc, char **argv)
{ {
@ -62,124 +64,128 @@ int main(int argc, char **argv)
argc--; argv++; argc--; argv++;
while (argc > 0 && **argv == '-') { while (argc > 0 && **argv == '-') {
(*argv)++; (*argv)++;
if (!(**argv)) if (!(**argv))
Usage(); Usage();
while (**argv) while (**argv)
switch (*(*argv)++) { switch (*(*argv)++) {
case 'v': op_flags |= OPT_v; break; case 'v': op_flags |= OPT_v; break;
default: Usage(); break; default: Usage(); break;
} }
argc--;argv++; argc--;argv++;
} }
string infile, outfile;
if (argc != 2) { if (argc == 2) {
Usage(); infile = *argv++;argc--;
outfile = *argv++;argc--;
Usage();
} else if (argc != 0) {
Usage();
} }
const char *infile = *argv++;argc--;
const char *outfile = *argv++;argc--;
string in; string in;
if (!file_to_string(infile, in)) { if (!file_to_string(infile, in)) {
cerr << "Cant read file\n" << endl; cerr << "Cant read file\n" << endl;
exit(1); exit(1);
} }
vector<unsigned int>ucsout1; vector<unsigned int>ucsout1;
string out, out1; string out, out1;
Utf8Iter it(in); Utf8Iter it(in);
FILE *fp = fopen(outfile, "w"); FILE *fp = 0;
if (fp == 0) { if (!outfile.empty()) {
fprintf(stderr, "cant create %s\n", outfile); fp = fopen(outfile.c_str(), "w");
exit(1); if (fp == 0) {
cerr << "Can't create " << outfile << endl;
exit(1);
}
} }
int nchars = 0; int nchars = 0;
for (;!it.eof(); it++) { for (;!it.eof(); it++) {
unsigned int value = *it; unsigned int value = *it;
if (value == (unsigned int)-1) { if (value == (unsigned int)-1) {
cerr << "Conversion error occurred\n" << endl; cerr << "Conversion error occurred at position " << it.getBpos()
exit(1); << endl;
} exit(1);
if (op_flags & OPT_v) { }
printf("Value: 0x%x", value); if (op_flags & OPT_v) {
if (value < 0x7f) fprintf(infout, "Value: 0x%04x", value);
printf(" (%c) ", value); if (value < 0x7f)
printf("\n"); fprintf(stdout, " (%c) ", value);
} fprintf(infout, "\n");
// UTF-32LE or BE array }
ucsout1.push_back(value); // UTF-32LE or BE array
// UTF-32LE or BE file ucsout1.push_back(value);
fwrite(&value, 4, 1, fp); if (fp) {
// UTF-32LE or BE file
fwrite(&value, 4, 1, fp);
}
// Reconstructed utf8 strings (2 methods) // Reconstructed utf8 strings (2 methods)
if (!it.appendchartostring(out)) if (!it.appendchartostring(out))
break; break;
// conversion to string // conversion to string
out1 += it; out1 += it;
// fprintf(stderr, "%s", string(it).c_str()); // fprintf(stderr, "%s", string(it).c_str());
nchars++; nchars++;
}
if (fp) {
fclose(fp);
} }
fclose(fp);
fprintf(stderr, "nchars %d\n", nchars); fprintf(infout, "Found %d Unicode characters\n", nchars);
if (in.compare(out)) { if (in.compare(out)) {
fprintf(stderr, "error: out != in\n"); fprintf(stderr, "error: out != in\n");
exit(1); exit(1);
} }
if (in != out1) { if (in != out1) {
fprintf(stderr, "error: out1 != in\n"); fprintf(stderr, "error: out1 != in\n");
exit(1); exit(1);
} }
// Rewind and do it a second time // Rewind and do it a second time
vector<unsigned int>ucsout2; vector<unsigned int>ucsout2;
it.rewind(); it.rewind();
for (int i = 0; ; i++) { for (int i = 0; ; i++) {
unsigned int value; unsigned int value;
if ((value = it[i]) == (unsigned int)-1) { if ((value = it[i]) == (unsigned int)-1) {
fprintf(stderr, "%d chars\n", i); break;
break; }
} it++;
it++; ucsout2.push_back(value);
ucsout2.push_back(value);
} }
if (ucsout1 != ucsout2) { if (ucsout1 != ucsout2) {
fprintf(stderr, "error: ucsout1 != ucsout2\n"); fprintf(stderr, "error: ucsout1 != ucsout2\n");
exit(1); exit(1);
} }
ucsout2.clear(); ucsout2.clear();
int ercnt; int ercnt;
const char *encoding = "UTF-32LE"; // note : use BE on high-endian machine const char *encoding = "UTF-32LE"; // note : use BE on high-endian machine
string ucs, ucs1; string ucs, ucs1;
for (vector<unsigned int>::iterator it = ucsout1.begin(); for (const unsigned int i : ucsout1) {
it != ucsout1.end(); it++) { ucs.append((const char *)&i, 4);
unsigned int i = *it;
ucs.append((const char *)&i, 4);
} }
if (!transcode(ucs, ucs1, if (!transcode(ucs, ucs1, encoding, encoding, &ercnt) || ercnt) {
encoding, encoding, &ercnt) || ercnt) { fprintf(stderr, "Transcode check failed, ercount: %d\n", ercnt);
fprintf(stderr, "Transcode check failed, ercount: %d\n", ercnt); exit(1);
exit(1);
} }
if (ucs.compare(ucs1)) { if (ucs.compare(ucs1)) {
fprintf(stderr, "error: ucsout1 != ucsout2 after iconv\n"); fprintf(stderr, "error: ucsout1 != ucsout2 after iconv\n");
exit(1); exit(1);
} }
if (!transcode(ucs, ucs1, if (!transcode(ucs, ucs1, encoding, "UTF-8", &ercnt) || ercnt) {
encoding, "UTF-8", &ercnt) || ercnt) { fprintf(stderr, "Transcode back to utf-8 check failed, ercount: %d\n",
fprintf(stderr, "Transcode back to utf-8 check failed, ercount: %d\n", ercnt);
ercnt); exit(1);
exit(1);
} }
if (ucs1.compare(in)) { if (ucs1.compare(in)) {
fprintf(stderr, "Transcode back to utf-8 compare to in failed\n"); fprintf(stderr, "Transcode back to utf-8 compare to in failed\n");
exit(1); exit(1);
} }
exit(0); exit(0);
} }