Added initial.

bin/convall.sh

@@ -0,0 +1,55 @@
+#!/bin/sh
+
+## bkimgproc - Book image processor
+## Copyright (c) 2012 Yoichi Imai, all rights reserved.
+
+SCRIPT_DIR="$(dirname -- "$(readlink -f -- "$0")")"
+BKIMGPROC_HOME="${BKIMGPROC_HOME:-${SCRIPT_DIR}/..}"
+. $BKIMGPROC_HOME/lib/bkimgproc.sh
+
+LOGFILE=$DATA_DIR/bkimgproc.log
+
+FINAL_STATUS=0
+bkimgproc_log "Start bkimgproc."
+for I in `ls -1d $DATA_INCOMING_DIR/*`; do
+  if [ ! -d "$I" ] ; then
+    bkimgproc_log "Not a directory. Skipped: $I"
+    continue
+  fi
+
+  BASE=`basename $I`
+  bkimgproc_log " Start converting $BASE..."
+
+  mv $I $DATA_WORKING_DIR/$BASE
+
+  $BKIMGPROC_HOME/bin/convone.sh $DATA_WORKING_DIR/$BASE >$DATA_WORKING_DIR/$BASE/convert.log 2>&1
+
+  STATUS=$?
+  if [ $STATUS -ne 0 ] ; then
+    FINAL_STATUS=$STATUS
+    bkimgproc_log " Failed to convert $BASE. See convert.log."
+    mv $DATA_WORKING_DIR/$BASE $DATA_FAILED_DIR/$BASE
+  else
+    bkimgproc_log " Succeeded to convert $BASE. Moving results..."
+
+    for J in $DATA_WORKING_DIR/$BASE/result/*; do
+      PLUGIN=`echo $J | sed -e 's/^.*_//' -e 's/\.[a-zA-Z0-9]*$//'`
+
+      if [ ! -d "$DATA_BOOKS_DIR/$PLUGIN" ] ; then
+        mkdir "$DATA_BOOKS_DIR/$PLUGIN"
+      fi
+
+      mv "$J" "$DATA_BOOKS_DIR/$PLUGIN/"
+    done
+
+    rmdir $DATA_WORKING_DIR/$BASE/result
+
+    # FIXME: duplicate check.
+    NEW_BASE=`date "+%Y%m%d%H%M%S"`-$BASE
+    bkimgproc_log " Backup project directory to $DATA_DONE_DIR/$NEW_BASE." 
+    mv $DATA_WORKING_DIR/$BASE $DATA_DONE_DIR/$NEW_BASE
+  fi
+done
+bkimgproc_log "Finished bkimgproc."
+
+exit $FINAL_STATUS

bin/convone.sh

@@ -0,0 +1,132 @@
+#!/bin/bash
+
+## bkimgproc - Book image processor
+## Copyright (c) 2012 Yoichi Imai, all rights reserved.
+set -e 
+
+if [ -z "$1" ] ; then
+  echo "usage: $0 proj-dir"
+  exit 1
+fi
+
+SCRIPT_DIR="$(dirname -- "$(readlink -f -- "$0")")"
+BKIMGPROC_HOME="${BKIMGPROC_HOME:-${SCRIPT_DIR}/..}"
+. $BKIMGPROC_HOME/lib/bkimgproc.sh
+
+PROJ_DIR="$1"
+
+TEMP_DIR=$DATA_TEMP_DIR/bkimgproc.$$
+mkdir $TEMP_DIR
+
+# Extract jpegs
+mkdir $TEMP_DIR/original
+
+find "$PROJ_DIR" -maxdepth 1 -type f | egrep -i "^.*/[0-9][-_0-9]*[a-z]*.(pdf|jpg)$" | while read FILE ; do
+  BASE="`basename $FILE`"
+  EXT="${BASE#*.}"
+  BASE="${BASE%.*}"
+
+  ## special treatment for '-' and '_': remove characters
+  BASE="`echo "$BASE" | sed -e 's/[-_]//g'`"
+
+  case "$EXT" in
+  jpg|JPG)
+    bkimgproc_log "Copy image from '$FILE'"
+    cp "$FILE" "$TEMP_DIR/original/$BASE-001.jpg"
+    ;;
+  *) 
+    bkimgproc_log "Extracting images from '$FILE'"
+
+    # pdfimages generates {base}-{num}.xxx
+    pdfimages -j $FILE "$TEMP_DIR/original/$BASE"
+  esac
+done
+
+bkimgproc_log "Calculating white levels"
+mkdir $TEMP_DIR/white
+$BKIMGPROC_HOME/bin/whiteleveldir.sh $TEMP_DIR/white $TEMP_DIR/original
+
+bkimgproc_log "Rotating images for 'r' or 'l' images"
+$BKIMGPROC_HOME/bin/rotate.sh $TEMP_DIR/original/$BASE/*
+
+mkdir $TEMP_DIR/work
+mkdir $TEMP_DIR/archive
+if [ ! -d "$PROJ_DIR/result" ] ; then
+  mkdir $PROJ_DIR/result
+fi
+
+for PLUGIN_DIR in $PLUGINS_DIR/*; do
+  # Converting 
+  PLUGIN=`basename $PLUGIN_DIR`
+  bkimgproc_log "Start converting for $PLUGIN"
+
+  DEST_DIR=$TEMP_DIR/work/$PLUGIN
+  mkdir $DEST_DIR
+
+  # ORIG_JPEG may be pbm
+  for ORIG_JPEG in `find $TEMP_DIR/original -type f | sort`; do
+    ORIG_JPEG_BASE=`basename $ORIG_JPEG`
+    # Remove extension
+    ORIG_JPEG_BASE=${ORIG_JPEG_BASE%.*}
+
+    PREFIX=${ORIG_JPEG_BASE%-*}
+    PAGE=${ORIG_JPEG_BASE#*-}
+    ODDEVEN=`expr $PAGE % 2` || true
+
+    WHITE=`cat $TEMP_DIR/white/white-$PREFIX-$ODDEVEN-center`
+    # [rl] is for the rotate option
+    OPTION=`echo $ORIG_JPEG_BASE | sed -e 's/^[0-9]*[rl]\?//' -e 's/-.*$//'`
+
+    if [ -z "$OPTION" ] ; then
+       sh -x $PLUGIN_DIR/options/_.sh $ORIG_JPEG $DEST_DIR/$ORIG_JPEG_BASE.jpg $WHITE
+    else
+       sh -x $PLUGIN_DIR/options/$OPTION.sh $ORIG_JPEG $DEST_DIR/$ORIG_JPEG_BASE.jpg $WHITE
+    fi
+  done
+
+  bkimgproc_log "Renaming images."
+  CUR=1
+  for OLD_PATH in `ls -1 $DEST_DIR/*.jpg | sort` ; do
+    NEW_NAME=`printf "P%05d.jpg" $CUR`
+    CUR=`expr $CUR + 1`
+
+    bkimgproc_log " Moving $OLD_PATH to $NEW_NAME" 
+    mv $OLD_PATH $DEST_DIR/$NEW_NAME
+  done
+
+  # Be careful: $RESULT_BASE may contain spaces.
+  bkimgproc_log "Finding name-pdf."
+  RESULT_BASE="`find "$PROJ_DIR" -name "*.pdf" -o -name "*.PDF" | sed -e 's/.*\///' -e 's/\.pdf$//i' | grep -v "^[0-9]"`"
+  if [ -z "$RESULT_BASE" ] ; then
+    RESULT_BASE="`basename "$PROJ_DIR"`"
+    bkimgproc_log "Not detected name pdf: use project basename"
+  fi
+  bkimgproc_log "Result name is '$RESULT_BASE'"
+
+  IS_R2L=""
+  if [ -f "$PROJ_DIR/r2l.txt" ] ; then
+    IS_R2L=1
+  fi
+  bkimgproc_log "Right-to-left option: $IS_R2L"
+
+  bkimgproc_log "Archiving for $PLUGIN"
+  mkdir $TEMP_DIR/archive/$PLUGIN
+  $PLUGIN_DIR/archive.sh $DEST_DIR $TEMP_DIR/archive/$PLUGIN "$IS_R2L"
+
+  # Check archive.* (archive.cbz, archive.pdf, etc)
+  ARCHIVE_FILE=`find $TEMP_DIR/archive/$PLUGIN -name "archive.*"`
+  if [ -z "$ARCHIVE_FILE" ] ; then
+    bkimgproc_log "Failed to archive for $PLUGIN"
+    exit 1 
+  fi
+
+  ARCHIVE_FILE_BASE="$(basename "$ARCHIVE_FILE")"
+  ARCHIVE_FILE_EXT="${ARCHIVE_FILE_BASE#*.}"
+  RESULT_FILE="$PROJ_DIR/result/${RESULT_BASE}_${PLUGIN}.${ARCHIVE_FILE_EXT}"
+  mv "$ARCHIVE_FILE" "$RESULT_FILE"
+
+  bkimgproc_log "Generated '$RESULT_FILE'"
+done
+
+rm -rf "$TEMP_DIR"
+

bin/rotate.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+## bkimgproc - Book image processor
+## Copyright (c) 2012 Yoichi Imai, all rights reserved.
+
+SCRIPT_DIR="$(dirname -- "$(readlink -f -- "$0")")"
+BKIMGPROC_HOME="${BKIMGPROC_HOME:-${SCRIPT_DIR}/..}"
+. $BKIMGPROC_HOME/lib/bkimgproc.sh
+
+for I in "$@"; do
+  BASE="`basename "$I"`"
+
+  ROTATE_OPTION="`echo "$BASE" | sed -e 's/^[0-9]*\(.\).*/\1/'`"
+
+  case "$ROTATE_OPTION" in
+  r)
+    EVEN_ANGLE=270
+    ODD_ANGLE=90
+    ;;
+  l)
+    EVEN_ANGLE=90
+    ODD_ANGLE=270
+    ;;
+  *)
+    continue
+  esac
+
+  PAGENUM="${BASE#*-}"
+  PAGENUM="${PAGENUM%.*}"
+
+  if [ `expr $PAGENUM % 2` -eq 0 ] ; then
+     bkimgproc_log "Rotate option $ROTATE_OPTION ($EVEN_ANGLE) $I"
+     mogrify -rotate $EVEN_ANGLE "$I"
+  else
+     bkimgproc_log "Rotate option $ROTATE_OPTION ($ODD_ANGLE) $I"
+     mogrify -rotate $ODD_ANGLE "$I"
+  fi
+done
+

bin/whitelevel.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+## bkimgproc - Book image processor
+## Copyright (c) 2012 Yoichi Imai, all rights reserved.
+
+if [ -z "$1" ]; then
+  echo usage: $0 image
+  exit 1
+fi
+
+SCRIPT_DIR="$(dirname -- "$(readlink -f -- "$0")")"
+BKIMGPROC_HOME="${BKIMGPROC_HOME:-${SCRIPT_DIR}/..}"
+. $BKIMGPROC_HOME/lib/bkimgproc.sh
+
+FILE="$1"
+INFOY=`convert -trim -fuzz 40% "$FILE" -format "%H %Y %h" info:`
+if [ -z "$INFOY" ]; then
+  echo -1
+  exit 0
+fi
+
+ORIG_HEIGHT=`echo $INFOY | awk '{print $1}'`
+TOP_BLANK=`echo $INFOY | awk '{print $2}' | tr -d '+'`
+CROPPED_HEIGHT=`echo $INFOY | awk '{print $3}'`
+
+# No blank detected. This page is all blank or gray.
+if [ "$TOP_BLANK" -eq -1 ] ; then
+  echo -1
+  exit 0
+fi
+
+BOTTOM_OFFSET=`expr $TOP_BLANK + $CROPPED_HEIGHT`
+BOTTOM_BLANK=`expr $ORIG_HEIGHT - $BOTTOM_OFFSET`
+
+TOP_BLANK_PERCENT=`expr $TOP_BLANK \* 100 / $ORIG_HEIGHT`
+BOTTOM_BLANK_PERCENT=`expr $BOTTOM_BLANK \* 100 / $ORIG_HEIGHT`
+
+# Too large: Just white or totally image. Too small: useless
+if [ $TOP_BLANK_PERCENT -le 1 -o $TOP_BLANK_PERCENT -ge 25 ] ; then
+  TOP_INVALID=1
+fi
+
+if [ $BOTTOM_BLANK_PERCENT -le 1 -o $BOTTOM_BLANK_PERCENT -ge 25 ] ; then
+  BOTTOM_INVALID=1
+fi
+
+# Both side are invalid, then this image is useless.
+if [ -n "$TOP_INVALID" -a -n "$BOTTOM_INVALID" ] ; then
+  echo -1
+  exit 0
+elif [ -n "$TOP_INVALID" ] ; then
+   RESULT="`convert "$FILE" -modulate 100,0 -crop x+0+$BOTTOM_OFFSET -format "%[mean] %[standard-deviation]" info:`"
+elif [ -n "$BOTTOM_INVALID" ] ; then
+   RESULT="`convert "$FILE" -modulate 100,0 -crop x$TOP_BLANK+0+0 -format "%[mean] %[standard-deviation]" info:`"
+else
+   RESULT="`convert "$FILE" -modulate 100,0 '(' -clone 0 -crop x$TOP_BLANK+0+0 ')' '(' -clone 0 -crop x+0+$BOTTOM_OFFSET ')' -delete 0 -append -format "%[mean] %[standard-deviation]" info:`"
+fi
+
+echo $RESULT | awk '{ print int(($1 - 3 * $2) / 655.35) }'
+

bin/whiteleveldir.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+## bkimgproc - Book image processor
+## Copyright (c) 2012 Yoichi Imai, all rights reserved.
+
+if [ -z "$2" ] ; then
+  echo usage: $0 tmp-white-dir images-dir
+  exit 1
+fi
+
+SCRIPT_DIR="$(dirname -- "$(readlink -f -- "$0")")"
+BKIMGPROC_HOME="${BKIMGPROC_HOME:-${SCRIPT_DIR}/..}"
+. $BKIMGPROC_HOME/lib/bkimgproc.sh
+
+TMP_WHITE_DIR="$1"
+IMAGES_DIR="$2"
+
+if [ ! -d "$TMP_WHITE_DIR" ] ; then
+  echo "TMP_WHITE_DIR: '$TMP_WHITE_DIR' is not created."
+  exit 1
+fi
+
+# Find images -999 (a naming rule for pdfimages) files.
+find $IMAGES_DIR -maxdepth 1 -type f | \
+  sed -ne 's/^\(.*\/\([^-]*\)-\([0-9][0-9][0-9]\)\.[^\.]*\)$/\1 \2 \3/p' | \
+  while read P PREFIX PAGE; do
+    ODDEVEN=`expr $PAGE % 2`
+    $BKIMGPROC_HOME/bin/whitelevel.sh $P >>$TMP_WHITE_DIR/white-$PREFIX-$ODDEVEN
+done
+
+# Calculate median
+for I in $TMP_WHITE_DIR/white-*; do
+  grep -v -- "-1" $I | grep -v "nan" >$I-valid
+  VALID_COUNT=`cat $I-valid | wc -l`
+  CENTER=`expr $VALID_COUNT / 2 + 1`
+  VAL=`sort -n $I-valid | head -$CENTER | tail -1`
+
+  # no value 
+  if [ -z "$VAL" ] ; then
+    SELECTED=$DEFAULT_WHITELEVEL
+  # too little sample
+  elif [ "$VALID_COUNT" -lt $WHITELEVEL_SAMPLES ]; then
+    SELECTED=$DEFAULT_WHITELEVEL
+  # too black
+  elif [ "$VAL" -lt $DANGER_WHITELEVEL ]; then
+    SELECTED=$DANGER_WHITELEVEL
+  else
+    SELECTED=$VAL
+  fi
+
+  echo $SELECTED>$I-center
+  echo "$I: valid-count,calculated,selected: $VALID_COUNT,$VAL,$SELECTED"
+done
+

lib/bkimgproc.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+PLUGINS_DIR="$BKIMGPROC_HOME/lib/plugins"
+
+DATA_DIR="$BKIMGPROC_HOME/data"
+
+DATA_INCOMING_DIR="$DATA_DIR/incoming"
+DATA_WORKING_DIR="$DATA_DIR/working"
+DATA_FAILED_DIR="$DATA_DIR/failed"
+DATA_DONE_DIR="$DATA_DIR/done"
+DATA_BOOKS_DIR="$DATA_DIR/books"
+DATA_TEMP_DIR="$DATA_DIR/temp"
+
+DEFAULT_WHITELEVEL=90
+DANGER_WHITELEVEL=70
+WHITELEVEL_SAMPLES=5
+
+# Common functions
+bkimgproc_log() {
+  DATE=`date "+%Y-%m-%d %H:%M:%S"`
+  echo "[$DATE] $1"
+}

lib/plugins/kobo/archive.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+set -e
+
+if [ -z "$2" ] ; then
+  echo usage: $0 temp-dest-dir temp-archive-dir
+  exit 1
+fi
+
+TEMP_DEST_DIR="$1"
+TEMP_ARCHIVE_DIR="$2"
+
+(cd "$TEMP_DEST_DIR" ; /usr/bin/zip $TEMP_ARCHIVE_DIR/archive.zip *.jpg )
+mv $TEMP_ARCHIVE_DIR/archive.zip $TEMP_ARCHIVE_DIR/archive.cbz

lib/plugins/kobo/options/_.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+set -e
+/usr/bin/convert "$1" -modulate 100,0 -level 10%,90%,0.6 -geometry 600x750 "$2"

lib/plugins/kobo/options/m.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+set -e
+
+if [ -z "$3" ] ; then
+  WHITE=90
+else
+  WHITE="$3"
+fi
+
+/usr/bin/convert -trim -fuzz 40% "$1" -modulate 100,0 -level 10%,${WHITE}% '(' +clone -roll +0+1 '(' +clone -roll +1-1 ')' +composite -compose Multiply ')' +composite -compose Multiply -gamma 0.5 -geometry 600x750 "$2"
+