Differences

This shows you the differences between two versions of the page.

--- code:shell:fetchpm25 [2015/01/20 21:44] – percy
+++ code:shell:fetchpm25 [2015/01/22 17:24] (current) – percy
@@ Line 5: / Line 5: @@
 So have fun.
+Before you can run it, you need install "dos2unix"
+  apt-get install dos2unix
@@ Line 17: / Line 20: @@
 OUTPUT_TMP="${TMP}/pm25_all_latest.txt"
 wget http://www.soupm25.com/ -O ${RAW_HTML}
-ALL_URLS=`cat ${RAW_HTML}|grep ".html"|sed "s/<a href=//g"|sed "s/\"//g"|sed "s/\/\///g"|sed "s/>//g"|sed "s/<\/a//g"|cut -d "=" -f4|sed "s/<\/li//g"|grep -v -E "^$|</html|DOCTYPE"|awk -F "www" '{print "http://www"$2}'|awk -F "html" '{print $1"html "$2}'|sed 's/  / /g'|dos2unix`
+ALL_URLS=`cat ${RAW_HTML}|grep ".html"|sed "s/<a href=//g"|sed "s/\"//g"|sed "s/\/\///g"|sed "s/>//g"|sed "s/<\/a//g"|cut -d "=" -f4|sed "s/<\/li//g"|grep -v -E "^$|</html|DOCTYPE"|awk -F "www" '{print "http://www"$2}'|awk -F "html" '{print $1"html "$2}'|sed 's/  / /g'|sort|uniq|dos2unix`
 ALL_CITY=`echo "${ALL_URLS}"|grep "city"`
 echo "${ALL_CITY}" > ${CITY}
@@ Line 31: / Line 34: @@
     i="${line}"
     url=`echo "${i}"|cut -d " " -f1`
-    city=`echo "${i}"|awk -F "city/" '{print $2}'|sed 's/.html /_/g'|dos2unix`
+    city=`echo "${i}"|awk -F "city/" '{print $2}'|sed 's/.html /_/g'`
     pingyin=`echo "${city}"|cut -d "_" -f1`
     #echo "${pingyin}"