recoll/src/filters/rclpdf

#!/bin/sh
# @(#$Id: rclpdf,v 1.10 2007-07-12 17:13:38 dockes Exp $  (C) 2004 J.F.Dockes
# This is copied almost verbatim from Estraier:
#================================================================
# Estraier: a personal full-text search system
# Copyright (C) 2003-2004 Mikio Hirabayashi
#================================================================
#================================================================
# Convert a pdf file to  HTML.
#
# We use pdftotxt from the xpdf package. This does not perfect results as
# whitespace is sometimes either arbitrarily inserted or stripped from the
# text. This seems to depend on the usage of option -raw, and,
# unfortunately also of the document itself, so that there does not seem to
# be an universally good solution
#
# Also, the filter sometimes seems to output problematic utf-8. I did not
# check if it was actually incorrect or just mis-understood by qtextedit
# (tobedone) 

# Uncomment the following if you get better results without. The
# pdftotext manual says that the option is no longer recommended The
# difference in output seems mostly the removal of soft-hyphens when
# -raw is not set
# optionraw=-raw

# set variables
LANG=C ; export LANG
LC_ALL=C ; export LC_ALL
progname="rclpdf"
filetype=pdf


#RECFILTCOMMONCODE
##############################################################################
# !! Leave the previous line unmodified!! Code imported from the
# recfiltcommon file

# Utility code common to all shell filters. This could be sourced at run
# time, but it's slightly more efficient to include the code in the
# filters at build time (with a sed script).

# Describe error in a way that can be interpreted by our caller
senderror()
{
    echo RECFILTERROR $*
    # Also alert on stderr just in case
    echo ":2:$progname::: $*" 1>&2
    exit 1
}

iscmd()
{
    cmd=$1
    case $cmd in
    */*)
	if test -x $cmd -a ! -d $cmd ; then return 0; else return 1; fi ;;
    *)
      oldifs=$IFS; IFS=":"; set -- $PATH; IFS=$oldifs
      for d in $*;do test -x $d/$cmd -a ! -d $d/$cmd && return 0;done
      return 1 ;;
    esac
}

checkcmds()
{
    for cmd in $*;do
      if iscmd $cmd 
      then 
        a=1
      else 
        senderror HELPERNOTFOUND $cmd
      fi
    done
}

# show help message
if test $# -ne 1 -o "$1" = "--help" 
then
  echo "Convert a $filetype file to HTML text for Recoll indexing."
  echo "Usage: $progname [infile]"
  exit 1
fi

infile="$1"

# check the input file existence (may be '-' for stdin)
if test "X$infile" != X- -a ! -f "$infile"
then
  senderror INPUTNOSUCHFILE "$infile"
fi

# protect access to our temp files and directories
umask 77

##############################################################################
# !! Leave the following line unmodified !
#ENDRECFILTCOMMONCODE

checkcmds pdftotext iconv awk

# Run pdftotext and fix the result (add a charset tag and fix the html escaping
# The strange 'BEGIN' setup is to prevent 'file' from thinking this file
# is an awk program
pdftotext $optionraw -htmlmeta -enc UTF-8 -eol unix -q "$infile" - |
iconv -f UTF-8 -t UTF-8 -c -s |
awk 'BEGIN'\
' {
  doescape = 0
  cont = ""
  charsetmeta = "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\">"
}
{
  $0 = cont $0
  cont = ""
  # Insert charset meta tag at end of header
  if(doescape == 0 && $0 ~ /<\/head>/) {
    match($0, /<\/head>/)
    part1 = substr($0, 0, RSTART-1)
    part2 = substr($0, RSTART, length($0))
    $0 =  part1 charsetmeta part2
  }
  if(doescape == 0 && $0 ~ /<title>.*<\/title>/){
    match($0, /<title>.*<\/title>/)
    part1 = substr($0, 0, RSTART-1)
    mid = substr($0, RSTART, RLENGTH)
    part2 = substr($0, RSTART + RLENGTH, length($0))
    gsub(/<title>/, "", mid)
    gsub(/<\/title>/, "", mid)
    gsub(/&/, "\\&amp;", mid)
    gsub(/</, "\\&lt;", mid)
    gsub(/>/, "\\&gt;", mid)
    mid = "<title>" mid "</title>"
    $0 = part1 mid part2
  }

  # Recoll treats "Subject" as a "title" element (based on emails). The PDF
  # "Subject" metadata field is more like an HTML "description"
  if(doescape == 0 && $0 ~ /<meta ?name="Subject"/){
      gsub(/="Subject"/, "=\"Description\"", $0)
  }

  if ($0 == "<pre>"){
    # Begin of body text. need to escape some chars from now on as 
    # pdftotext sometimes doesnt do it
    doescape++
    print $0
    next
  } else if ($0 ~ /<\/pre>/){
    doescape--
    print $0 
    next
  } else if ($0 ~ /[­-]$/) {
    # Note : soft-hyphen is iso8859 0xad
    # Break at last whitespace
    match($0, "[ \t][^ \t]+$")
    line = substr($0, 0, RSTART)
    cont = substr($0, RSTART, RLENGTH-1)
    $0 = line
    # print "LINE [" $0 "] CONT[" cont "]"
  } else if($0 == "\f"){
    $0 = "<hr>"
    print 
    next
  }
  if(doescape > 0){
      gsub(/&/, "\\&amp;", $0)
      gsub(/</, "\\&lt;", $0)
      gsub(/>/, "\\&gt;", $0)
  }
  print $0
}
'