spacepaste

Paste details

reply | raw | toggle line wrapping

  1.  
  2. from bs4 import BeautifulSoup
  4. import requests
  5. import urllib2
  6. #url = raw_input("Enter a website to extract the URL's from: ")
  8. import tempfile
  9. import os
  10. import json
  11. import copy
  13. from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
  14. from pdfminer.pdfpage import PDFPage
  15. from pdfminer.converter import TextConverter
  16. from pdfminer.layout import LAParams
  17. from cStringIO import StringIO
  20. from pypdfocr.pypdfocr import PyPDFOCR
  22. import urllib
  24. def get_sheriff_sale_data(url):
  25. r = requests.get(url)
  27. data = r.text
  29. soup = BeautifulSoup(data)
  31. properties = []
  32. propertyTableRows = soup.find_all('th', '', 'Sale date')[0].parent.parent.find_all('tr')
  33. propertyTableRows = propertyTableRows[1:] #get rid of the table header
  35. #[1].find_all('td')[3].text
  36. for tr in propertyTableRows:
  37. tdSibs = tr.find_all('td')
  38. if len(tdSibs) == 5:
  39. property = {}
  40. property['date'] = tdSibs[0].text
  41. property['property_type'] = tdSibs[1].text
  42. property['city'] = tdSibs[2].text
  43. property['notice_of_sale'] = tdSibs[3].text
  44. property['pdf'] = tdSibs[3].find('a').get('href')
  45. property['supporting_docs'] = [a.get('href') for a in tdSibs[4].find_all('a')]
  46. properties.append(property)
  47. return properties
  49. def remove_filedata_from_properties(properties):
  50. fileData=[]
  51. fileName ='sherrifSalePropertiesParsed.json'
  52. #
  53. #open the existing file if it exists
  54. if os.path.isfile(fileName):
  55. with open(fileName, 'r') as jsonFile:
  56. try:
  57. fileData = json.load(jsonFile)
  58. except:
  59. pass
  60. for item in fileData:
  61. #to prevent re-downloading PDFs, remove any current properties that were in the file
  62. if item in properties:
  63. properties.remove(item)
  65. def get_json_data(fileName):
  66. fileData=[]
  67. #open the existing file if it exists
  68. if os.path.isfile(fileName):
  69. with open(fileName, 'r') as jsonFile:
  70. try:
  71. fileData = json.load(jsonFile)
  72. except:
  73. pass
  74. return fileData
  76. def update_json_with_downloaded_pdfs(properties):
  77. fileName ='sherrifSalePropertiesParsed.json'
  78. fileData = get_json_data(fileName)
  79. for item in properties:
  80. #to prevent re-downloading PDFs, remove any current properties that were in the file
  81. if item not in fileData:
  82. fileData.append(item)
  83. with open(fileName, 'w+') as jsonFile:
  84. jsonFile.write(json.dumps(fileData))
  86. def save_json(properties):
  87. fileName ='latestProperties.json'
  88. with open(fileName, 'w+') as jsonFile:
  89. jsonFile.write(json.dumps(properties))
  92. def make_pdf_dir():
  93. try:
  94. dirPath = os.path.abspath('pdfs')
  95. os.makedirs(dirPath)
  96. except:
  97. print "error with making pdfs dir in this folder, trying tempfolder instead: %s" % tempfile.gettempdir()
  98. dirPath = os.path.abspath(os.path.join(tempfile.gettempdir(), 'pdfs'))
  99. os.makedirs(dirPath)
  100. return dirPath
  102. def download_property_pdfs(properties, siteUrl, dir):
  103. for i, property in enumerate(properties):
  104. pdfUrl = property['pdf']
  105. name = pdfUrl.split('/')[-1]
  106. pdfUrl = siteUrl + pdfUrl
  107. rq = urllib2.Request(pdfUrl)
  108. res = urllib2.urlopen(rq)
  109. pdf = open(os.path.join(dir, name), 'wb+')
  110. pdf.write(res.read())
  111. pdf.close()
  112. print "PERCENT DONE DOWNLOADING PDFs: %s%%" % str(float(i+1)/len(properties)*100)
  115. def convert_pdf(path):
  116. rsrcmgr = PDFResourceManager()
  117. retstr = StringIO()
  118. codec = 'utf-8'
  119. laparams = LAParams()
  120. device = TextConverter(rsrcmgr, retstr, codec=codec, laparams=laparams)
  121. #
  122. fp = file(path, 'rb')
  123. #process_pdf(rsrcmgr, device, fp)
  124. interpreter = PDFPageInterpreter(rsrcmgr, device)
  125. for page in PDFPage.get_pages(fp, range(10),
  126. maxpages=5, password='',
  127. caching=True, check_extractable=False):
  128. #page.rotate = (page.rotate+0) % 360
  129. interpreter.process_page(page)
  130. #
  131. fp.close()
  132. device.close()
  133. #
  134. str = retstr.getvalue()
  135. retstr.close()
  136. return str
  138. def get_all_PDF_addresses(properties, dir):
  139. for i, property in enumerate(properties):
  140. if hasattr(property, 'address'):
  141. continue
  142. try:
  143. pdfUrl = property['pdf']
  144. name = pdfUrl.split('/')[-1]
  145. #
  146. nameNoExt = name[:-4]
  147. nameOCR = nameNoExt + '_ocr.pdf'
  148. print dir, name
  149. path = os.path.abspath(os.path.join(dir, name))
  150. #path.replace('\\\\','\\')
  151. #path1 = '"%s"' % path
  152. #use PDFminer to get extract the text
  153. text = convert_pdf(path)
  154. #if the PDFminer text didn't find our string, try doing OCR
  155. if not 'The property is commonly known as' in text:
  156. #path.replace('\\\\','\\')
  157. #path1 = '"%s"' % path
  158. do_PDF_OCR(path)
  159. path = os.path.abspath(os.path.join(dir, nameOCR))
  160. text = convert_pdf(path)
  161. if not 'The property is commonly known as' in text:
  162. print 'OCR failed'
  163. sys.exit(0)
  164. #tsplit = text.split('\n')
  165. idx = text.index('commonly known')
  166. tsplit = text[idx:idx+150].split('\n')
  167. #addr1 = tsplit[idx].split(':')[1].strip()
  168. addr1 = tsplit[2].strip()
  169. addr2 = tsplit[4].strip()
  170. properties[i]['address'] = addr1 + ', ' + addr2
  171. except Exception as e:
  172. print "\n Error during parsing %s" % repr(e)
  173. pass
  174. print "\nPERCENT DONE: %s%%" % str(float(i+1)/len(properties)*100)
  176. def do_PDF_OCR(path):
  177. script = PyPDFOCR()
  178. print path
  179. script.go([path])
  183. googleGeocodeUrl = 'http://maps.googleapis.com/maps/api/geocode/json?'
  185. def get_coordinates(query, from_sensor=False):
  186. query = query.encode('utf-8')
  187. params = {
  188. 'address': query,
  189. 'sensor': "true" if from_sensor else "false"
  190. }
  191. url = googleGeocodeUrl + urllib.urlencode(params)
  192. json_response = urllib.urlopen(url)
  193. response = json.loads(json_response.read())
  194. if response['results']:
  195. location = response['results'][0]['geometry']['location']
  196. latitude, longitude = location['lat'], location['lng']
  197. print query, latitude, longitude
  198. else:
  199. latitude, longitude = None, None
  200. print query, "<no results>"
  201. return latitude, longitude
  205. def convertDictToGeoJSON(properties, fileName='geoJsonSheriffs.geojson'):
  206. geoList = []
  207. hits=0
  208. misses=0
  209. for property in properties:
  210. if not 'address' in property:
  211. misses+=1
  212. continue
  213. try:
  214. coords = (list(get_coordinates(property['address'])))[::-1]
  215. except UnicodeDecodeError:
  216. misses+=1
  217. continue
  218. if None in coords or 'null' in coords:
  219. misses+=1
  220. continue
  221. geoDict = {}
  222. geoDict['type'] = 'Feature'
  223. geoDict["geometry"]= {
  224. "type": "Point",
  225. "coordinates": coords
  226. }
  227. geoDict["properties"]={
  228. "name": property['address'] + ' --- \n' + property['notice_of_sale'] + ' --- \n' + property['date']
  229. }
  230. geoList.append(geoDict)
  231. hits+=1
  232. print "SUCCESS with errors possible: %d, FAIL: %d" % (hits, misses)
  233. with open(fileName, 'w+') as jsonFile:
  234. jsonFile.write(json.dumps(geoList))
  236. #import os
  237. #path = os.path.abspath(r"c:\Users\Chandni\Appdata\local\temp\pdfs\8-15Herscovitz.pdf")
  239. #get_all_PDF_addresses(properties, dirPath)
  240. #do_PDF_OCR(fileName)
  242. #fileText = convert_pdf()
  246. if __name__=='__main__':
  247. mainUrl = 'http://www.oregonsheriffs.com/'
  248. wacoUrl = mainUrl + 'sales-washington.htm'
  249. web_properties = get_sheriff_sale_data(wacoUrl)
  250. #
  251. properties_needing_downloaded=copy.deepcopy(web_properties)
  252. remove_filedata_from_properties(properties_needing_downloaded)
  254. #dirPath = make_pdf_dir()
  255. dirPath = os.path.abspath(os.path.join(tempfile.gettempdir(), 'pdfs'))
  256. download_property_pdfs(properties_needing_downloaded, mainUrl, dirPath)
  257. update_json_with_downloaded_pdfs(properties_needing_downloaded)
  258. file_properties = get_json_data('sherrifSalePropertiesParsed.json')
  259. try:
  260. get_all_PDF_addresses(file_properties, dirPath)
  261. finally:
  262. update_json_with_downloaded_pdfs(file_properties)
  263.  
Powered by spacepaste. Paste removal.