/[safari]/get_book.sh
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /get_book.sh

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.1 by dpavlin, Sun Dec 14 19:11:30 2003 UTC revision 1.7 by dpavlin, Sun Feb 15 11:40:43 2004 UTC
# Line 1  Line 1 
1  #!/bin/sh  #!/bin/sh
2    
3  #export http_proxy=http://proxy.pliva.hr:8080  # proxy settings (same as in firebird)
4    fping proxy && export http_proxy=http://proxy:8080
5    # user agent (same as in firebird)
6    ua="Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.5) Gecko/20031225 Firebird/0.7"
7    
8    # wait between pages
9    export wait=120
10    
11    
12    if [ -z "$1" ] ; then
13            echo "Usage: $0 ISBN"
14            exit 1
15    fi
16    
17    if [ -e orig ] ; then
18            echo "orig directory found. Resume download? [Y/n]"
19            read ans
20            if [ "$ans" = "n" ] ; then
21                    exit 1;
22            fi
23            mv orig/* .
24            rm -Rf orig
25    
26            grep -l 'promo.asp' * | xargs -i rm {}
27            grep -l 'This is only a preview of the full book' * | xargs -i rm {}
28    fi
29    
30  #isbn="0-201-41975-0"  isbn=$1;
 isbn="0-672-32240-4"  
   
 wait=10  
31    
32  isbn2=`echo $isbn | sed 's/-//g'`  isbn2=`echo $isbn | sed 's/-//g'`
33    
34  function mirror() {  function mirror() {
35    
36            url="$1"
37    
38            file=`echo $url | sed -e s,http://[^?]*?,index.html?, -e s,#.*$,, -e s,/,%2F,g`
39            if [ -e "$file" ] ; then
40    #               echo "skip $url"
41                    return
42            fi
43    
44            if echo $url | grep '/index' >/dev/null ; then
45                    cookies=""
46                    echo -n "no login (index) "
47            elif echo $url | grep 'mode=toc' >/dev/null ; then
48                    cookies=""
49                    echo -n "no login (toc) "
50            else
51                    cookies="--load-cookies=/home/dpavlin/.phoenix/default/g6b45nt6.slt/cookies.txt"
52                    echo -n "with login "
53            fi
54            echo $url
55    
56          wget -p -nH -nc -k \          wget -p -nH -nc -k \
57                  --random-wait --wait=$wait -t 0 \                  -t 1 -U "$ua" \
58                  --load-cookies=/home/dpavlin/.phoenix/default/g6b45nt6.slt/cookies.txt \                  $cookies $url
                 -U "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.5) Gecko/20031206 Firebird/0.7" \  
                 $1  
59    
60  #       -D safari.oreilly.com \  #       -D safari.oreilly.com \
61  #       -A 0-201-41975-0 \  #       -A 0-201-41975-0 \
62    
63            perl -e '$t=rand($ENV{wait} || 30);print "sleep for $t sec.\n"; sleep($t);'
64  }  }
65    
66  function geturl() {  function geturl() {
67          hindent -s $1 | grep $2 | grep -i href | grep mode=[st][eo]c | \          hindent -s $1 | grep -i href | grep mode=[st][eo]c | \
68          sed -e 's/^.*<a.*href="//i' \          sed -e 's/^.*<a.*href="//i' \
69                  -e 's/".*//' -e 's/amp;//g' \                  -e 's/".*//' -e 's/amp;//g' \
70                  -e 's,^[^\?]*\?,http://safari.oreilly.com/,' \                  -e 's,^[^\?]*\?,http://safari.oreilly.com/,' \
71                  -e 's/#$//' \                  -e 's/#$//' \
72                  -e 's/srchText=//' | \                  -e 's/\&srchText=//' \
73                  grep -v open=false | \                  -e 's/open=false/open=true/' | \
                 grep -v 'view=[A-Z].*%2F[^i]' | \  
                 grep -v 'view=[A-Z].*/[^i]' | \  
74                  grep '&s=1&b=1&f=1&t=1&c=1&u=1&r=&o=1' | \                  grep '&s=1&b=1&f=1&t=1&c=1&u=1&r=&o=1' | \
75                    grep $2 | \
76                    grep -v "$2/[0-9][0-9][0-9][0-9][0-9][0-9][0-9]" | \
77                  sort -u >> in                  sort -u >> in
78  }  }
79    
80  echo > in  function uniqurl() {
81  #mirror "http://safari.oreilly.com/?XmlId=$isbn"          mv in in.tmp
82            grep -v 'view=[A-Z]' in.tmp | sort -u > in
83            grep 'view=[A-Z].*/index' in.tmp | sort -u >> in
84    }
85    
86    function mirror_in() {
87            cat in | while read url ; do
88                    mirror "$url"
89                    #sleep $wait
90    
91                    if grep 'promo.asp' `ls -t index.html* | head -3` >/dev/null ; then
92                            echo "WARNING: safari seems to logout you as user. Aborting."
93                            exit 1
94                    fi
95    
96                    if grep -i '>Account locked<' `ls -t index.html* | head -3` >/dev/null ; then
97                            echo "WARNING: your safari account is locked. Aborting."
98                            exit 1
99                    fi
100    
101                    if grep -i 'session disabled' `ls -t index.html* | head -3` >/dev/null ; then
102                            echo "WARNING: your safari session is disabled. Aborting."
103                            exit 1
104                    fi
105    
106                    if grep -i 'This is only a preview of the full book' `ls -t index.html* | head -3` >/dev/null ; then
107                            echo "WARNING: you didn't add this book to your bookshelf!"
108                            exit 1
109                    fi
110            done
111    }
112    
113    echo -n > in
114    mirror "http://safari.oreilly.com/?XmlId=$isbn"
115    
116  echo "extract URLs from first page..."  echo "extract URLs from first page..."
117  geturl "index.html?XmlId=$isbn" $isbn  geturl "index.html?XmlId=$isbn" $isbn
118    uniqurl
119    
120  mirror "-i in"  mirror_in
121    
122  echo -n "extracting URLs [1]"  echo -n "extracting URLs [1]"
123  ls index.html* | while read file ; do  ls index.html* | while read file ; do
# Line 49  ls index.html* | while read file ; do Line 126  ls index.html* | while read file ; do
126  done  done
127  echo  echo
128    
129  sort -u in > in2  uniqurl
130    
131  mirror "-i in2"  mirror_in
132    
133  echo > in  echo -n > in
134  echo -n "extracting URLs [2]"  echo -n "extracting URLs [2]"
135  ls index.html* | while read file ; do  ls index.html* | while read file ; do
136          echo -n "."          echo -n "."
137          geturl $file $isbn          geturl $file $isbn
138  done  done
139    
140  sort -u in > in2  uniqurl
141    
142    mirror_in
143    
144    # convert links in html
145    bn=`basename $0`
146    dir=`echo $0 | sed "s/$bn$//"`
147    ls index.html* | xargs -i $dir/filter.pl {}
148    mkdir orig
149    mv index.html* orig/
150    
 mirror "-i in2"  
Legend:
Removed from v.1.1  
changed lines
  Added in v.1.7
  ViewVC Help
Powered by ViewVC 1.1.26