CrunchBang Linux Pastebin - collaborative debugging

pastebin is a collaborative debugging tool allowing you to share and modify code snippets while chatting on IRC, IM or a message board.

This site is developed to XHTML and CSS2 W3C standards. If you see this paragraph, your browser does not support those standards and you need to upgrade. Visit WaSP for a variety of options.

CrunchBang Linux Pastebin

Posted by Anonymous on Sat 5th Nov 00:56 (modification of post by luc view diff)
diff | download | new post

  1. #!/bin/bash
  2. #+FIXME /bin/bash is needed for the array in get_next_manga_animea_net() and in
  3. #+the preparations part for a test with [[ ]]
  4. # variables: ##################################################################
  5. # JOB can be "all", "preload" or "img"
  6. LOGFILE=manga.log
  7. RESUME=false
  8. JOB=all
  9. WGET_OPTION=--no-verbose
  10. CURL_OPTION='--write-out "Done: %{url_effective}"'
  11. QUIET=false
  12. START="Started '$0${@:+ $@}' at `date "+%F %H:%M:%S"`"
  13. MANGADIR=${MANGADIR:-~/manga}
  14. # functions: ##################################################################
  15. help_function () {
  16.   local PROG=`basename "$0"`
  17.   echo "usage: $PROG [ -xq ] [ -d dir | -f log ] URL"
  18.   echo "       $PROG -r [ -d dir | -f log ]"
  19.   echo "       $PROG [ -f log ] -c arcive"
  20.   echo "dir defaults to \`.' and log defaults to \`manga.log'"
  21.   echo "normal behavior is to load the images starting with the one embedged in URL"
  22.   echo "-r will resume from a logfile (for example if new chapters where published)"
  23.   echo "-c will cat the logfile inside an archive (not very sufisticated)"
  24. }
  25. if which -s wget; then
  26.   load_to_file () { wget $WGET_OPTION --output-document="$2" "$1"; }
  27.   load_to_pipe () { wget --quiet --output-document=- "$1"; }
  28. elif which -s curl; then #TODO buggy
  29.   load_to_file () { curl --silent $CURL_OPTION --output "$2" "$1"; }
  30.   load_to_pipe () { curl --silent --output - "$1"; }
  31. elif which -s elinks; then
  32.   load_to_file () { elinks -source "$1" > "$2"; }
  33.   load_to_pipe () { elinks -source "$1"; }
  34. else
  35.   echo "Don't know how to load data (wget/curl not found)" >&2
  36.   exit 1
  37. fi
  38. # a function for every known host to fetch the next URL, IMG and FILENAME.
  39. get_next_www_mangafox_com () {
  40.   if [ -z "$URL" ]; then exit -1 ; fi
  41.   NEXT=`load_to_pipe "$URL" | sed -n '/return enlarge/{s/.*href="\([^"]*\)".*src="\([^"]*\)".*/\1 \2/p;}'`
  42.   IMG=${NEXT#* }
  43.   FILE=${URL%.html}.${IMG##*.}
  44.   FILE=${FILE##*/}
  45.   if [[ ${NEXT%% *} = "javascript:void(0);" ]]; then #TODO
  46.    # echo "!void(0)!"
  47.     #exit -1
  48.     NEXT=
  49.   else #TODO
  50.     NEXT=${URL%/*.html}/${NEXT%% *}
  51.   fi #TODO
  52. }
  53. get_next_www_mangareader_net () {
  54.   NEXT=`load_to_pipe "$URL" | sed -n '/imgholder/{N;s/.*href="\([^"]*\)".*src="\([^"]*\)".*alt="\([^"]*\)".*/\2"\1"\3/p;q;}'`
  55.   IMG=${NEXT%%\"*}
  56.   FILE="${NEXT##*\"}.${IMG##*.}"
  57.   NEXT=${NEXT#*\"}
  58.   NEXT=http://www.mangareader.net${NEXT%\"*}
  59.   if [ "$IMG" ]; then return 0; else return 1; fi
  60. }
  61. get_next_manga_animea_net () { #TODO test w/o array
  62.   eval LINE=(`load_to_pipe "$URL" | sed -n '/imagelink/{s/ /%20/g;s/.*href="\([^"]*\)".*src="\([^"]*\)".*/\1 \2/;p;}'`)
  63.   IMG=${LINE[1]}
  64.   #IMG=${LINE#* }
  65.   FILE="${URL%%.html}.${IMG##*.}"
  66.   FILE=${FILE##*/}
  67.   NEXT=${LINE[0]}
  68.   #NEXT=${LINE% *}
  69.   if [[ ${#LINE[@]} -eq 2 ]]; then return 0; else return 1; fi
  70. }
  71. get_next_read_homeunix_com () {
  72.   if [ -z "$URL" ]; then return 1; fi
  73.   LINE=`load_to_pipe "$URL" | grep "document.write.*\(IMG ALT\)\|\(NEXT CHAPTER\)"`
  74.   IMG=`sed -n '/SRC/{;s/.*SRC="\([^"]*\).*/\1/;s/ /%20/g;p;}' <<<"$LINE"`
  75.   if [[ "$LINE" = *NEXT\ CHAPTER* ]]; then
  76.     NEXT=`sed -n '/NEXT CHAPTER/{s#.*href ="\([^"]*\).*#http://read.homeunix.com/onlinereading/\1#;s/ /%20/g;p;}' <<<"$LINE"`
  77.   else
  78.     NEXT=`sed -n '/IMG ALT/{s/.*href ="\([^"]*\)".*/\1/;s/ /%20/g;p;}' <<<"$LINE"`
  79.   fi
  80.   #FIXME: What filename to use?
  81.   FILE=$((++i)).${IMG##*.}
  82.   return 0
  83. }
  84. cleanup_interrupt () {
  85.   :
  86. }
  87. # options: ####################################################################
  88. while getopts c:d:f:hj:rxq FLAG; do
  89.   case $FLAG in
  90.     c) tar --wildcards -xOf "$OPTARG" "*/$LOGFILE"; exit;;
  91.     d) DIR=$OPTARG;;
  92.     f) LOGFILE=`basename "${OPTARG}"`; DIR=`dirname "${OPTARG}"`;;
  93.     h) help_function; exit 1;;
  94.     j) JOB=$OPTARG;;
  95.     r) RESUME=true;;
  96.     x) set -x;;
  97.     q) WGET_OPTION=--quiet CURL_OPTION=--silent QUIET=true;;
  98.     \?) exit 43;;
  99.   esac
  100. done
  101. shift $((OPTIND-1))
  102. # preparation: ################################################################
  103. if [ "$DIR" ]; then
  104.   if [[ $DIR != */* ]]; then DIR=$MANGADIR/$DIR; fi
  105.   if mkdir -p "$DIR"; then cd "$DIR"; else exit -1; fi
  106.   $QUIET || echo "Working in $PWD"
  107. fi
  108. if $RESUME; then
  109.   URL=`tail -n 1 "$LOGFILE"`
  110.   URL=${URL%% *}
  111. else
  112.   URL="$1"
  113. fi
  114. if [ -z "$URL" ]; then echo "No URL given. Try '${0##*/} -h' for help."; exit 1; fi
  115. GET_NEXT=${URL#http://}
  116. GET_NEXT=${GET_NEXT%%/*}
  117. GET_NEXT=get_next_${GET_NEXT//./_}
  118. if $RESUME; then $GET_NEXT; fi
  119. # work: #######################################################################
  120. case $JOB in
  121.   all)
  122.     while $GET_NEXT; do
  123.       load_to_file "$IMG" "$FILE" &
  124.       #wget $WGET_OPTION --output-document="$FILE" "$IMG" &
  125.       echo "$URL $IMG" >> "$LOGFILE"
  126.       URL=$NEXT
  127.       unset NEXT
  128.     done
  129.     ;;
  130.   preload)
  131.     while $GET_NEXT; do
  132.       echo "$URL $IMG" >> "$LOGFILE"
  133.       echo "Currently loading $URL"
  134.       URL=$NEXT
  135.       unset NEXT
  136.     done
  137.     ;;
  138.   img)
  139.     while read URL IMG; do
  140.       #TODO
  141.       #echo "Not implemented yet!" >&2
  142.       #exit -1
  143.       echo "URL=$URL"
  144.       echo "IMG=$IMG"
  145.     done < "$LOGFILE"
  146.     ;;
  147.   *) echo "Not a valid job name!"; exit 1;;
  148. esac
  149. wait
  150. if ! $QUIET; then
  151.   echo "$START"
  152.   date +"%F %H:%M:%S Images loaded. Exiting ..."
  153. fi

Submit a correction or amendment below (click here to make a fresh posting)
After submitting an amendment, you'll be able to view the differences between the old and new posts easily.

Syntax highlighting:

To highlight particular lines, prefix each line with @@


Remember me