/[safari]/get_book.sh
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /get_book.sh

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.1 by dpavlin, Sun Dec 14 19:11:30 2003 UTC revision 1.8 by dpavlin, Tue Feb 17 19:06:10 2004 UTC
# Line 1  Line 1 
1  #!/bin/sh  #!/bin/sh
2    
3  #export http_proxy=http://proxy.pliva.hr:8080  # proxy settings (same as in firebird)
4    fping proxy && export http_proxy=http://proxy:8080
5    # user agent (same as in firebird)
6    ua="Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.5) Gecko/20031225 Firebird/0.7"
7    
8    # wait between pages
9    export wait=120
10    
11    
12    if [ -z "$1" ] ; then
13            echo "Usage: $0 ISBN"
14            exit 1
15    fi
16    
17    if [ -e orig ] ; then
18            echo "orig directory found. Resume download? [Y/n]"
19            read ans
20            if [ "$ans" = "n" ] ; then
21                    exit 1;
22            fi
23            mv orig/* .
24            rm -Rf orig
25    
26            grep -l 'promo.asp' * | xargs -i rm {}
27            grep -l 'This is only a preview of the full book' * | xargs -i rm {}
28    fi
29    
30  #isbn="0-201-41975-0"  isbn=$1;
 isbn="0-672-32240-4"  
   
 wait=10  
31    
32  isbn2=`echo $isbn | sed 's/-//g'`  isbn2=`echo $isbn | sed 's/-//g'`
33    
34  function mirror() {  function mirror() {
35    
36            url="$1"
37    
38            file=`echo $url | sed -e s,http://[^?]*?,index.html?, -e s,#.*$,, -e s,/,%2F,g`
39            if [ -e "$file" ] ; then
40    #               echo "skip $url"
41                    echo -n "."
42                    return
43            fi
44    
45            if echo $url | grep '/index' >/dev/null ; then
46                    cookies=""
47                    echo -n "no login (index) "
48            elif echo $url | grep 'mode=toc' >/dev/null ; then
49                    cookies=""
50                    echo -n "no login (toc) "
51            else
52                    cookies="--load-cookies=/home/dpavlin/.phoenix/default/g6b45nt6.slt/cookies.txt"
53                    echo -n "with login "
54            fi
55            echo $url
56    
57          wget -p -nH -nc -k \          wget -p -nH -nc -k \
58                  --random-wait --wait=$wait -t 0 \                  -t 1 -U "$ua" \
59                  --load-cookies=/home/dpavlin/.phoenix/default/g6b45nt6.slt/cookies.txt \                  $cookies $url
                 -U "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.5) Gecko/20031206 Firebird/0.7" \  
                 $1  
60    
61  #       -D safari.oreilly.com \  #       -D safari.oreilly.com \
62  #       -A 0-201-41975-0 \  #       -A 0-201-41975-0 \
63    
64            perl -e '$t=rand($ENV{wait} || 30);print "sleep for $t sec.\n"; sleep($t);'
65  }  }
66    
67  function geturl() {  function geturl() {
68          hindent -s $1 | grep $2 | grep -i href | grep mode=[st][eo]c | \          hindent -s $1 | grep -i href | grep mode=[st][eo]c | \
69          sed -e 's/^.*<a.*href="//i' \          sed -e 's/^.*<a.*href="//i' \
70                  -e 's/".*//' -e 's/amp;//g' \                  -e 's/".*//' -e 's/amp;//g' \
71                  -e 's,^[^\?]*\?,http://safari.oreilly.com/,' \                  -e 's,^[^\?]*\?,http://safari.oreilly.com/,' \
72                  -e 's/#$//' \                  -e 's/#.*$//' \
73                  -e 's/srchText=//' | \                  -e 's/\&srchText=//' \
74                  grep -v open=false | \                  -e 's/open=false/open=true/' | \
                 grep -v 'view=[A-Z].*%2F[^i]' | \  
                 grep -v 'view=[A-Z].*/[^i]' | \  
75                  grep '&s=1&b=1&f=1&t=1&c=1&u=1&r=&o=1' | \                  grep '&s=1&b=1&f=1&t=1&c=1&u=1&r=&o=1' | \
76                    grep $2 | \
77                    grep -v "$2/[0-9][0-9][0-9][0-9][0-9][0-9][0-9]" | \
78                  sort -u >> in                  sort -u >> in
79  }  }
80    
81  echo > in  function uniqurl() {
82  #mirror "http://safari.oreilly.com/?XmlId=$isbn"          mv in in.tmp
83            grep -v 'view=[A-Z]' in.tmp | sort -u > in
84            grep 'view=[A-Z].*/index' in.tmp | sort -u >> in
85            links=`wc -l in | cut -d" " -f1`
86            echo "found $links unique links"
87    }
88    
89    function mirror_in() {
90            cat in | while read url ; do
91                    mirror "$url"
92                    #sleep $wait
93    
94                    if grep 'promo.asp' `ls -t index.html* | head -3` >/dev/null ; then
95                            echo "WARNING: safari seems to logout you as user. Aborting."
96                            exit 1
97                    fi
98    
99                    if grep -i '>Account locked<' `ls -t index.html* | head -3` >/dev/null ; then
100                            echo "WARNING: your safari account is locked. Aborting."
101                            exit 1
102                    fi
103    
104                    if grep -i 'session disabled' `ls -t index.html* | head -3` >/dev/null ; then
105                            echo "WARNING: your safari session is disabled. Aborting."
106                            exit 1
107                    fi
108    
109                    if grep -i 'This is only a preview of the full book' `ls -t index.html* | head -3` >/dev/null ; then
110                            echo "WARNING: you didn't add this book to your bookshelf!"
111                            exit 1
112                    fi
113            done
114            echo
115    }
116    
117    echo -n > in
118    mirror "http://safari.oreilly.com/?XmlId=$isbn"
119    echo
120    
121  echo "extract URLs from first page..."  echo "extract URLs from first page..."
122  geturl "index.html?XmlId=$isbn" $isbn  geturl "index.html?XmlId=$isbn" $isbn
123    uniqurl
124    
125  mirror "-i in"  mirror_in
126    
127  echo -n "extracting URLs [1]"  echo -n "extracting URLs [1]"
128  ls index.html* | while read file ; do  ls index.html* | while read file ; do
# Line 49  ls index.html* | while read file ; do Line 131  ls index.html* | while read file ; do
131  done  done
132  echo  echo
133    
134  sort -u in > in2  uniqurl
135    
136  mirror "-i in2"  mirror_in
137    
138  echo > in  echo -n > in
139  echo -n "extracting URLs [2]"  echo -n "extracting URLs [2]"
140  ls index.html* | while read file ; do  ls index.html* | while read file ; do
141          echo -n "."          echo -n "."
142          geturl $file $isbn          geturl $file $isbn
143  done  done
144    
145  sort -u in > in2  uniqurl
146    
147    mirror_in
148    
149    # convert links in html
150    bn=`basename $0`
151    dir=`echo $0 | sed "s/$bn$//"`
152    ls index.html* | xargs -i $dir/filter.pl {}
153    mkdir orig
154    mv index.html* orig/
155    
 mirror "-i in2"  
Legend:
Removed from v.1.1  
changed lines
  Added in v.1.8
  ViewVC Help
Powered by ViewVC 1.1.26