tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

utf8iter driver: read from stdin

Browse Source
This commit is contained in:
Jean-Francois Dockes 2019-07-05 11:33:29 +02:00
parent 75759ed481
commit 3f33d1d0ea
1 changed files with 87 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
src/testmains/trutf8iter.cpp
View File

@ -56,6 +56,8 @@ void Usage() {
static int op_flags;
#define OPT_v 0x2
FILE *infout = stdout;
int main(int argc, char **argv)
{
thisprog = argv[0];
@ -73,12 +75,14 @@ int main(int argc, char **argv)
}
argc--;argv++;
}
if (argc != 2) {
string infile, outfile;
if (argc == 2) {
infile = *argv++;argc--;
outfile = *argv++;argc--;
Usage();
} else if (argc != 0) {
Usage();
}
const char *infile = *argv++;argc--;
const char *outfile = *argv++;argc--;
string in;
if (!file_to_string(infile, in)) {
cerr << "Cant read file\n" << endl;
@ -88,29 +92,35 @@ int main(int argc, char **argv)
vector<unsigned int>ucsout1;
string out, out1;
Utf8Iter it(in);
FILE *fp = fopen(outfile, "w");
FILE *fp = 0;
if (!outfile.empty()) {
fp = fopen(outfile.c_str(), "w");
if (fp == 0) {
fprintf(stderr, "cant create %s\n", outfile);
cerr << "Can't create " << outfile << endl;
exit(1);
}
}
int nchars = 0;
for (;!it.eof(); it++) {
unsigned int value = *it;
if (value == (unsigned int)-1) {
cerr << "Conversion error occurred\n" << endl;
cerr << "Conversion error occurred at position " << it.getBpos()
<< endl;
exit(1);
}
if (op_flags & OPT_v) {
printf("Value: 0x%x", value);
fprintf(infout, "Value: 0x%04x", value);
if (value < 0x7f)
printf(" (%c) ", value);
printf("\n");
fprintf(stdout, " (%c) ", value);
fprintf(infout, "\n");
}
// UTF-32LE or BE array
ucsout1.push_back(value);
if (fp) {
// UTF-32LE or BE file
fwrite(&value, 4, 1, fp);
}
// Reconstructed utf8 strings (2 methods)
if (!it.appendchartostring(out))
@ -121,9 +131,11 @@ int main(int argc, char **argv)
// fprintf(stderr, "%s", string(it).c_str());
nchars++;
}
if (fp) {
fclose(fp);
}
fprintf(stderr, "nchars %d\n", nchars);
fprintf(infout, "Found %d Unicode characters\n", nchars);
if (in.compare(out)) {
fprintf(stderr, "error: out != in\n");
exit(1);
@ -139,7 +151,6 @@ int main(int argc, char **argv)
for (int i = 0; ; i++) {
unsigned int value;
if ((value = it[i]) == (unsigned int)-1) {
fprintf(stderr, "%d chars\n", i);
break;
}
it++;
@ -155,13 +166,10 @@ int main(int argc, char **argv)
int ercnt;
const char *encoding = "UTF-32LE"; // note : use BE on high-endian machine
string ucs, ucs1;
for (vector<unsigned int>::iterator it = ucsout1.begin();
it != ucsout1.end(); it++) {
unsigned int i = *it;
for (const unsigned int i : ucsout1) {
ucs.append((const char *)&i, 4);
}
if (!transcode(ucs, ucs1,
encoding, encoding, &ercnt) || ercnt) {
if (!transcode(ucs, ucs1, encoding, encoding, &ercnt) || ercnt) {
fprintf(stderr, "Transcode check failed, ercount: %d\n", ercnt);
exit(1);
}
@ -170,8 +178,7 @@ int main(int argc, char **argv)
exit(1);
}
if (!transcode(ucs, ucs1,
encoding, "UTF-8", &ercnt) || ercnt) {
if (!transcode(ucs, ucs1, encoding, "UTF-8", &ercnt) || ercnt) {
fprintf(stderr, "Transcode back to utf-8 check failed, ercount: %d\n",
ercnt);
exit(1);
@ -182,4 +189,3 @@ int main(int argc, char **argv)
}
exit(0);
}