#!/usr/bin/python3
"""
Lawsoc Scraper
# 1. Submit a letter to the search page
# 2. test that we get fewer results than 500 (apply xpath to get the results #
# number)
# 3. if small than 500 calculate the range results/20 for for loop to fetch all
# 4. if the returned number is greater than 500 add another letter to the search
# 5. else continue to stepping through the results to get the id, link and lat-lon
"""

from subprocess import check_output
from io import StringIO
from urllib.parse import urlparse
from lxml import etree
from lxml.html import parse
import string
import os
import time
import requests
import math
import json
import vobject
import datetime
# from lxml.html import tostring
# from lxml.html import HtmlMixin
# from io import BytesIO
# from urlparse import urlparse
# from stem import Signal
# from stem.control import Controller
# from stem.util import term
# from geopy.geocoders import Nominatim
# import shutil
# import signal
# import requesocks
# import stem.process
# import uuid


# Output vcard:
# Using vobject construct a vcard for output. The issue is that the fields
# supported by thunderbird/icedove are fixed to the fields it displays and
# the original vcard information is lost.
# sogo connector only supports these fields. See scraper2carddav-outline.txt

# look up using first line of address and lip (last line) to get full address
# back, return full dict
def osm_address_lookup(dict, geolocator):
    """ Returns OSM address, latitude and longitude. """
    streetname = "+".join(dict.address[0:2])
    postcode = dict.address[-2]

    address, (latitude, longitude) = geolocator.geocode({
        'street': streetname,
        'postalcode': postcode
    })

    return address, latitude, longitude


# the dict passed in (output_person_details-person_dict{},
# output_office_details-office_dict{}) should be of the form read from a valid
# vcf vcard by vobject.
def create_vcard(dict):
    """ Return a serialized vcard. """
    new_vcard = vobject.vCard()
    line = new_vcard.add('n')
    line.n.value = vobject.vcard.Name(family=dict.familyName, given=dict.givenName)

    line = new_vcard.add('fn')
    line.fn.value = dict.givenName + ' ' + dict.familyName

    line = new_vcard.add('org')
    line.org.value = dict['work_office']

    # line = new_vcard.add('nickname')
    # line.nickname.value =  dict.nickname

    # unlimited number of separate values, map to list with foreach areas of practice
    if 'Areas_of_Practice' in dict:
        line = new_vcard.add('categories')
        for category in dict['Areas_of_Practice']:
            line.categories.value = category  # multiple line from list in dict, needs a loop FIXME

            # six lines, but the last 4 are city, state, zip and country for thunderbird
            # Reverse index for the last 5 lines, concatenate any remaining in first line.
            line = new_vcard.add('adr')
            line.type_param = "work"
            line.adr.value = ",".join(
                dict.address[
                    0:len(
                        dict.work_address
                    )-5
                ]
            )  # FIXME
            line.adr.value = dict.address[-5]  # line two
            line.adr.value = dict.address[-4]  # city
            line.adr.value = dict.address[-3]  # state
            line.adr.value = dict.address[-2]  # zip
            line.adr.value = dict.address[-1]  # country

            # six lines, but the last 4 are city, state, zip and country for thunderbird
            # line = new_vcard.add('adr')
            # line.type_param = "home"
            # line.adr.value =  dict. # line one
            # line.adr.value =  dict. # line two
            # line.adr.value =  dict. # city
            # line.adr.value =  dict. # state
            # line.adr.value =  dict. # zip
            # line.adr.value =  dict. # country

            line = new_vcard.add('tel')
            line.tel.type_param = "work"
            line.tel.value = dict.tel_work

            # line = new_vcard.add('tel')
            # line.tel.type_param = "home"
            # line.tel.value =  dict.tel_home

            # line = new_vcard.add('tel')
            # line.tel.type_param = "cell"
            # line.tel.value =  dict.tel_cell

            line = new_vcard.add('tel')
            line.tel.type_param = "fax"
            line.tel.value = dict.tel_fax

            # line = new_vcard.add('tel')
            # line.tel.type_param = "pager"
            # line.tel.value =  dict.tel_pager

            # line = new_vcard.add('x-mozilla-html')
            # line.value =  False

            line = new_vcard.add('email')
            line.email.type_param = "work"
            line.email.value = dict.email_work

            # line = new_vcard.add('email')
            # line.email.type_param = "home"
            # line.email.value =  dict.email_home

            line = new_vcard.add('url')
            line.email.type_param = "work"
            line.email.value = dict.url_work

            # line = new_vcard.add('url')
            # line.url.type_param = "home"
            # line.url.value =  dict.url_home

            # line = new_vcard.add('title')
            # line.title.value =  dict.title

            # Maybe reuse for date admitted or add an extra field
            # line = new_vcard.add('bday')
            # line.bday.value = '1970-01-01'
            # line.bday.value =  person_dict.bday

            # custom value 1-4
            # use for sra_id
            line = new_vcard.add('custom1')
            line.custom1.value = dict.sra_id

            # use for dx_address
            line = new_vcard.add('custom2')
            line.custom2.value = dict.dx_address

            # use for type
            line = new_vcard.add('custom3')
            line.custom3.value = dict.type

            line = new_vcard.add('custom4')
            # line.custom4.value =  dict.corresponding value FIXME

            # single value each separated with "\n"
            line = new_vcard.add('note')
            if 'areas_of_practice' in dict:
                line.note.value = "\n".join(dict.areas_of_practice)
            else:
                line.note.value = "\n".join(dict.facilities)

            line = new_vcard.add('geo')
            if 'location' in dict:
                line.geo.value = ";".join(dict.location)
            # else:
            #    Pass

                # FIXME
                # line = new_vcard.add('x-aim')
                # line.x-aim.value = dict.aim
                #
                # # generate uuid from lawsoc id number
                # line = new_vcard.add('x-abuid')
                # if 'person_id' in dict:
                # 	line.x-abuid.value =  uuid.uuid3(uuid.NAMESPACE_DNS, dict.person_id)
                # else:
                # 	line.x-abuid.value =  uuid.uuid3(uuid.NAMESPACE_DNS, dict.solicitor_id)
                #
                line = new_vcard.add('rev')
                # line.value =  2015-11-25T23:17:35+00:00 #use date function
                now = datetime.datetime.now()
                line.rev.value = now.isoformat()

                # # morefunctionsforAB-TB3 fields
                # # tab additional, 5 available slots
                # line = new_vcard.add('email')
                # line.email.type_param = "internet"
                # line.email.value =

                # FIXME
                # other data tab - aniversary, spouse, last modified, 8 extra
                # categories
                line = new_vcard.add('x-anniversary')
                # line.x-anniversay.value =

                line = new_vcard.add('x-spouse')
                # line.x-spouse.value =

                line = new_vcard.add('categories')
                line.categories.type_param = "charset=utf"

                # line.categories.value =

                return new_vcard.serialize()


def get_pid(name):
    """ Returns the process id of the argument passed. """
    return list(map(int, check_output(["pidof", name]).split()))[0]


def convert_year(date):
    d, m, y = date.split('/')
    return "/".join([m, d, '%d%s' % (19 if int(y) >= 50 else 20, y), m, d])


def fetch_url(url):
    """ Return the text from a url. """
    # text = fetch_url_requesocks(url)
    text = fetch_url_requests(url)
    return text


def fetch_url_requests(url):
    """ Return the text of a url (using a proxy). """
    print("fetch_url_requests: Sending request with requests...")
    # squid3
    squid3_proxies = {
            "http": "http://127.0.0.1:3128",
            "https": "http://127.0.0.1:3128"
    }
    # polipo
    polipo_proxies = {
            "http": "http://127.0.0.1:8123",
            "https": "http://127.0.0.1:8123"
    }
    print("fetch_url_requests: url: " + url)
    r = requests.get(url, proxies=polipo_proxies)
    text = r.text
    print("fetch_url_requests: text: " + text)
    return text


# def fetch_url_requesocks(url):
#   """ Return the text of a url (using requesocks). """
#    print("(+) Sending request with requesocks...")
#    session = requesocks.session()
#    # tor browser bundle
#    session.proxies = {'http': 'socks5://127.0.0.1:9150',
#                       'https': 'socks5://127.0.0.1:9150'}
#    print(url)
#    r = session.get(url)
#    text = r.text
#    return text


# FIXME
# def tor_start():
#   """ Start the tor process. """
#    print(term.format("Starting Tor:\n", term.Attr.BOLD))
#
#    tor_process = stem.process.launch_tor_with_config(
#            config = {
#                'SocksPort': str(SOCKS_PORT),
#                'ExitNodes': '{ru}',
#                },
#            init_msg_handler = print_bootstrap_lines,
#            )
#    print(term.format("\nChecking our endpoint:\n", term.Attr.BOLD))
#    return tor_process

def tor_kill(tor_process):
    """ Kills the tor process """
    tor_process.kill()


def tor_stop(controller):
    """ Stops the tor controller. """
    controller.close()


# FIXME
# def tor_new_circuit(controller, port=9051, password=None):
#    """ Requests a new circuit from tor. """
#    with Controller.from_port(port) as controller:
#        controller.authenticate(password)
#        controller.signal(Signal.NEWNYM)
#
#
# def tor_reload(port=9051, password=None):
#    """ Reload tor. """
#    with Controller.from_port(port) as controller:
#        controller.authenticate(password)
#        controller.signal(Signal.HUP)


def polipo_status():
    """ Checks the status of Polipo. """
    '''
    evals to a boolean - FIXME xpath applies in firepath showing status change be not here. Stays false.
    '''
    url = 'http://127.0.0.1:8123/polipo/status?'
    tree = make_simple_tree(url)
    status_xpath = '//text() = "The shared proxy on sparky:8123 is on line."'
    status_result = apply_xpath(tree, status_xpath)
    if status_result:
        print("status_page: Polipo is online.")
    return status_result


def polipo_on():
    """ Turns on Polipo. """
    # /usr/bin/curl -m 5 -d 'proxyOffline=false' 'http://127.0.0.1:8123/polipo/config?'
    print("******************************************************************************************************")
    print("------------------------------------------------------------------------------------------------------")
    print('polipo_on: Turning Polipo on.')
    url = "http://127.0.0.1:8123/polipo/config?"
    data = 'proxyOffline=false'
    response = requests.post(url, data=data)
    # print("Polipo response.json: " + str(response.json))
    # print("Polipo response.text: " + str(response.text))
    print("Polipo response.status_code: " + str(response.status_code))
    # print("Polipo status: " + str(polipo_status()))


def polipo_off():
    """ Turns off Polipo. """
    # /usr/bin/curl -m 5 -d 'proxyOffline=true' 'http://127.0.0.1:8123/polipo/config?'
    print('polipo_off: Turning Polipo off.')
    print("------------------------------------------------------------------------------------------------------")
    print("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++")
    url = "http://127.0.0.1:8123/polipo/config?"
    data = 'proxyOffline=true'
    response = requests.post(url, data=data)
    # print("Polipo response.json: " + str(response.json))
    # print("Polipo response.text: " + str(response.text))
    print("Polipo response.status_code: " + str(response.status_code))
    # print("Polipo status: " + str(polipo_status()))
    return response.status_code


# def make_results_tree(name, page):
#    """ Returns a tree - takes a name to search for and a page number and returns an etree suitable for apply_xpath() """
#    url = "http://solicitors.lawsociety.org.uk/search/results?Name=%s&Type=0&IncludeNlsp=false&Pro=True&Page=%s" % (name, page)
#    text = fetch_url(url)
#    #print(text)
#    tree = parse(StringIO(text))
#    return tree


def make_results_tree(name, page):
    """ Returns a tree - takes a name to search for and a page number and returns an etree suitable for apply_xpath() """
    url = "http://solicitors.lawsociety.org.uk/search/results?Name=%s&Type=0&IncludeNlsp=false&Pro=True&Page=%s" % (name, page)
    with open(lawsoc_results_fo, 'r') as fo:
        if not any(url == x.rstrip('\r\n') for x in fo):
            print("New Lawsoc results page URL, writing to file." + " - " + url)
            write_file(lawsoc_results_fo, url)
        else:
            print("Lawsoc results page URL already saved.")

    text = fetch_url(url)
    # print(text)
    tree = parse(StringIO(text))
    return tree


def make_office_url(classification, idnumber, options):
    """ Returns a URL - takes a office/person, id options after a ? and returns a url. """
    url = "http://solicitors.lawsociety.org.uk/%s/%s/%s" % (classification, idnumber, options)
    # print("make_office_url: url: " + url)
    return url


def make_office_tree(classification, idnumber, options):
    """ Returns a tree - takes a office/person, id options after a ? and returns an etree suitable for apply_xpath(). """
    url = "http://solicitors.lawsociety.org.uk/%s/%s/%s" % (classification, idnumber, options)
    # print("make_office_tree: url: " + url)
    text = fetch_url(url)
    tree = parse(StringIO(text))
    return tree


def make_simple_tree(url):
    """ Returns a tree - Fetches the content from the given URL
    takes a url and returns a tree suitable for apply_xpath()"""
    text = fetch_url(url)
    # print(text)
    tree = parse(StringIO(text))
    return tree


def apply_xpath(tree, results_xpath):
    """ Applies the xpath to the result. """
    xPath = etree.XPath(results_xpath)
    results = xPath(tree)
    print("apply_xpath: results xpath: " + str(results_xpath))
    print("apply_xpath: results type: " + str(type(results)))
    print("apply_xpath: results:  " + str(results))
    return results


def apply_xpath_attribute(tree, results_xpath, attribute="alt"):
    """ The xpath will return a snipit that also has to be turned into a tree before the result. """
    xPath = etree.XPath(results_xpath)
    results = xPath(tree)
    print("apply_xpath_findall: results xpath: " + str(results_xpath))
    print("apply_xpath_findall: results type: " + str(type(results)))
    print("apply_xpath_findall: results:  " + str(results))
    for item in results:
        print(item)
        print(item.get(attribute))
    return results


def get_numberOfResults(name, page):
    """ Returns the number od results. """
    print("get_numberOfResults: name: " + name)
    print("get_numberOfResults: page: " + str(page))
    my_tree = make_results_tree(name, page)
#    results_xpath = "string(.//*[@class='search-results-controls']/div[2]/h1/strong[1])"
#    results_xpath = ".//*[@class='search-results-controls']/div[2]/h1/strong[1]/text()"
#    results_xpath = ".//*[@id='main-content']//h1[contains(.,'Found')]/strong[1]/text()"
    results_xpath = "string(.//*[@id='main-content']//h1[contains(.,'Found')]/strong[1])"
    results = apply_xpath(my_tree, results_xpath)
    print("get_numberOfResults: Value of results before casting to int: " + str(results[0]))
    print("get_numberOfResults: Length of results before casting to int: " + str(len(results[0])))
    print("get_numberOfResults: Type of results before casting to int: " + str(type(results[0])))
    results = int(results[0])
    return results


# def office_id(snip):
""" Returns office id number. """
#    #print("office_id: snip: " + str(type(snip)))
#    idnumber = snip.split("/")[2]
#    return idnumber


def office_id(url):
    """ Returns the office id. """
    office_url_parsed = urlparse(url)
    office_id = office_url_parsed.path.split('/')[2]
    return office_id


def office_classification(snip):
    """ Returns the office classification. """
    # print("office_classification: snip: " + str(type(snip)))
    classification = snip.split("/")[1]
    return classification


def office_url(tree):
    """ Returns a list of relative URLS - Takes a tree. """
    office_url_xpath = "\
            .//*[@class='solicitor solicitor-type-firm ']/header/h2/a/@href |\
            .//*[@class='solicitor solicitor-type-firm']/header/h2/a/@href |\
            .//*[@class='solicitor solicitor-type-firm last']/header/h2/a/@href"
    office_urls = apply_xpath(tree, office_url_xpath)
    return office_urls


def person_url(tree):
    """ Returns a list of relative urls - Takes a tree."""
    person_url_xpath = "\
    .//*[@class='solicitor solicitor-type-individual ']/header/h2/a/@href |\
    .//*[@class='solicitor solicitor-type-individual']/header/h2/a/@href |\
    .//*[@class='solicitor solicitor-type-individual last']/header/h2/a/@href"
    person_urls = apply_xpath(tree, person_url_xpath)
    return person_urls


def office_make_absolute(office_url):
    """ Returns an absolute URL for lawsoc - Takes a relative URL. """
    idnumber = office_id(office_url)
    classification = office_classification(office_url)
    options = ""
    # print(idnumber)
    # print(classification)
    url = make_office_url(classification, idnumber, options)
    return url


def people_id(snip):
    """ Returns the office id. """
    # print("office_id: snip: " + str(type(snip)))
    snip = snip.strip()
    idnumber = snip.split("/")[3]
    return idnumber


def people_classification(snip):
    """ Returns the people classification """
    # print("office_classification: snip: " + str(type(snip)))
    snip = snip.strip()
    classification = snip.split("/")[1] + "/" + snip.split("/")[2]
    return classification


def people_office_id(snip):
    """ Return the people office id. """
    print("------------------------------------------------------------------------------")
    print("people_office_id: snip: " + str(type(snip)))
    print("people_office_id: snip: " + str(snip))
    snip = snip.strip()
    print("people_office_id: snip.strip(): " + str(snip))
    # idnumber = snip.split("/")[3]
    idnumber = snip.split("/")[-2]
    return idnumber


def people_make_absolute(people_url):
    """ Takes a relative url and returns an absolute for lawsoc """
    print("people_make_absolute: people_url: " + str(people_url))
    idnumber = people_office_id(people_url)
    classification = people_classification(people_url)
    options = "?Solicitors=True"
    print(idnumber)
    print(classification)
    url = make_office_url(classification, idnumber, options)
    return url


def office_name(tree):
    """ Returns the office name. """
    office_name_xpath = "string(.//*[@id='main-content']/article/header/h1)"
    office_name = apply_xpath(tree, office_name_xpath)
    office_name = office_name
    return office_name


def person_telephone_number(tree):
    """ Returns the person telephone number. """
    telephone_xpath = "string(.//*[@id='main-details-accordion']/div/dl/dt[text()='Tel:']/following-sibling::dd[1])"
    telephone = apply_xpath(tree, telephone_xpath)
    return telephone


def office_telephone_number(tree):
    """ Returns the office telephone number. """
    telephone_xpath = "string(.//*[@id='main-details-accordion']/dl/dt[text()='Tel:']/following-sibling::dd[1])"
    telephone = apply_xpath(tree, telephone_xpath)
    return telephone


def office_website_url(tree):
    """ Return the office website. """
    website_xpath = "string(.//*[@id='main-details-accordion']/dl[1]/dt[text()='Web:']/following-sibling::dd[1]/a/@href)"
    website = apply_xpath(tree, website_xpath)
    return website


def person_admission_date(tree):
    """ Returns the person admission date. """
    admission_date_xpath = "string(.//*[@id='main-content']/article/header/p/span)"
    admission_date = apply_xpath(tree, admission_date_xpath)
    admission_date = admission_date.strip('Admitted as a solicitor: ')
    admission_date = convert_year(admission_date)
    return admission_date


def person_office_name(tree):
    """ Returns the person office name. """
    office_name_xpath = "string(string(.//*[@id='main-details-accordion']/div/dl[1]/dd[1]/a[1]))"
    office_name = apply_xpath(tree, office_name_xpath)
    office_name = office_name.strip('@')
    return office_name


def person_role(tree):
    """ Returns the person role. """
    role_xpath = "string(.//*[@id='main-details-accordion']/div[1]/dl[1]/dt)"
    role = apply_xpath(tree, role_xpath)
    role = role.split(" ")[0]
    return role


def person_email_address(tree):
    """ Returns person email address. """
    email_xpath = "string(.//*[@id='main-details-accordion']/div/dl/dt[text()='Email:']/../dd/a)"
    email = apply_xpath(tree, email_xpath)
    # email = email.strip('mailto')
    return email


def office_email_address(tree):
    """ Returns office email address. """
    email_xpath = "string(.//*[@id='main-details-accordion']/dl/dt[text()='Email:']/../dd/a)"
    email = apply_xpath(tree, email_xpath)
    # email = email.strip('mailto')
    return email


def office_latitude(tree, idnumber):
    """ Returns office latitude. """
    lat_xpath = "string(.//*[@id='solicitor%s']/@data-longitude)" % idnumber
    latitude = apply_xpath(tree, lat_xpath)
    return latitude


def office_longitude(tree, idnumber):
    """ Returns office longitude. """
    long_xpath = "string(.//*[@id='solicitor%s']/@data-latitude)" % idnumber
    longitude = apply_xpath(tree, long_xpath)
    return longitude


def person_solicitorID(tree):
    """ Returns person solicitor id. """
    # solicitorID_xpath = ".//*[@id='main-details-accordion']/div[1]/dl[1]/dd[1]/a[1]/@href"
    solicitorID_xpath = ".//*[@id='main-details-accordion']//a[contains(@href,'office')]/@href"
    url = apply_xpath(tree, solicitorID_xpath)[0]
    parsed_url = urlparse(url)
    solicitorID = parsed_url.path.split('/')[2]
    return solicitorID


def person_DX_address(tree):
    """ Returns person DX address. """
    person_DX_address_xpath = "string(.//*[@id='main-details-accordion']/dl[2]/dd[2])"
    person_DX_address = apply_xpath(tree, person_DX_address_xpath)
    return person_DX_address


def person_id(url):
    """ Returns person id. """
    person_url_parsed = urlparse(url)
    person_id = person_url_parsed.path.split('/')[2]
    return person_id


def person_sraID(tree):
    """ Returns person sra ID. """
    sraID_xpath = "string(.//*[@id='main-details-accordion']/dl[1]/dt[text()='SRA ID:']/following-sibling::dd[1])"
    sraID = apply_xpath(tree, sraID_xpath)
    return sraID


def person_accreditations(tree):
    """ Returns person accrediration. """
    print("-------------------------------------------------------------------------------------------------------")
    person_accreditations_xpath = ".//*[@class='accreditation']/@alt"
    person_accreditations_raw = apply_xpath(tree, person_accreditations_xpath)
    print("person_accreditations_raw " + str(person_accreditations_raw))
    print("type person_accreditations_raw " + str(type(person_accreditations_raw)))
    person_accreditations = person_accreditations_raw
    return person_accreditations


def person_areas_of_practice(tree):
    """ Returns person area of practice. """
    person_areas_of_practice_xpath = "string(.//*[@id='areas-of-practice-accordion']/ul)"
    person_areas_of_practice_raw = apply_xpath(tree, person_areas_of_practice_xpath)
    person_areas_of_practice = [x.strip() for x in person_areas_of_practice_raw.split('\r\n')]
    person_areas_of_practice = list(filter(None, person_areas_of_practice))
    return person_areas_of_practice


def person_languages(tree):
    """ Returns person language. """
    person_languages_xpath = "string(.//*[@id='languages-spoken-accordion']/ul)"
    person_languages_raw = apply_xpath(tree, person_languages_xpath)
    person_languages = [x.strip() for x in person_languages_raw.split('\r\n')]
    person_languages = list(filter(None, person_languages))
    return person_languages


def office_sraID(tree):
    """ Returns office sra ID. """
    sraID_xpath = "string(.//*[@id='main-details-accordion']/dl[1]/dt[text()='SRA ID:']/following-sibling::dd[1])"
    sraID = apply_xpath(tree, sraID_xpath)
    return sraID


def office_DX_address(tree):
    """ Return office DX address. """
    DX_address_xpath = "string(.//*[@id='main-details-accordion']/dl[2]/dd[2])"
    DX_address = apply_xpath(tree, DX_address_xpath)
    DX_address = DX_address.strip()
    return DX_address


def office_solicitor_type(tree):
    """ Returns office solicitor type. """
    office_solicitor_type_xpath = "string(.//*[@id='main-details-accordion']/dl/dt[text()='Type:']/following-sibling::dd[1])"
    office_solicitor_type = apply_xpath(tree, office_solicitor_type_xpath)
    return office_solicitor_type


def office_address(tree):
    """ Return office address. """
    address_xpath = "normalize-space(.//*[@id='main-details-accordion']/dl/dt[text()='Address:']/following-sibling::dd[1])"
    address = apply_xpath(tree, address_xpath)
    print(type(address))
    address = address.strip(' View in Google Maps').split(", ")
    return address


def office_people_url(tree):
    """ Returns office people url. """
    office_people_url_xpath = "string(.//*[@id='people-and-structure-accordion']/div[1]/ul/li[2]/a/@href)"
    office_people_url = apply_xpath(tree, office_people_url_xpath)
    office_people_url = people_make_absolute(office_people_url)
    return office_people_url


def office_facilities(tree):
    """ Return office facilities. """
    # office_facilities_xpath = "string(.//*[@id='main-details-accordion']/div/p/img/@data-original-title)"
    office_facilities_xpath = ".//*[@id='main-details-accordion']/div/p/img/@data-original-title"
    office_facilities = apply_xpath(tree, office_facilities_xpath)
    # now create a list from the xpath results FIXME
    print(office_facilities)
    office_facilities_list = office_facilities
    return office_facilities_list


def busy_page(tree):
    """ Returns busy page - evals to a boolean """
    busy_page_xpath = "//text() = 'Server Too Busy'"
    busy_page_result = apply_xpath(tree, busy_page_xpath)
    if busy_page_result:
        print("busy_page: Server Too Busy.")
    return busy_page_result


def error_page(tree):
    """ Returns error page - evals to a boolean """
    # error_page_xpath = ".//*[@id='main-content']/h2/text()"
    error_page_xpath = "//text() = 'Sorry, server error'"
    error_page_result = apply_xpath(tree, error_page_xpath)
    if error_page_result:
        print("error_page: Sorry, server error.")
    return error_page_result


def polipo_error(tree):
    """ Returns polipo error page - evals to a boolean """
    server_timeout_xpath = "//text() = '502 Read from server failed: Timeout'"
    server_dropped_xpath = "//text() = '502 Server dropped connection'"
    missing_page_xpath = "//text() = '502 Disconnected operation and object not in cache'"
    socks_failure_xpath = "//text() = '504 Connect to solicitors.lawsociety.org.uk:80 failed: General SOCKS server failure'"
    server_timeout_result = apply_xpath(tree, server_timeout_xpath)
    server_dropped_result = apply_xpath(tree, server_dropped_xpath)
    missing_page_result = apply_xpath(tree, missing_page_xpath)
    socks_failure_result = apply_xpath(tree, socks_failure_xpath)
    if server_timeout_result:
        print("polipo_error: Read from server failed: Timeout")
        return server_timeout_result
    elif server_dropped_result:
        print("polipo_error: 502 Server dropped connection")
        return server_dropped_result
    elif missing_page_result:
        print("polipo_error: 502 Disconnected operation and object not in cache")
        return missing_page_result
    elif socks_failure_result:
        print("polipo_error: 504 General SOCKS server failure")
        return socks_failure_result


def kill_tor():
    """ Usually don't have permission to do this and does not restart FIXME """
    # tor_pid = get_pid("tor")
    # os.kill(tor_pid, signal.SIGHUP)
    # print("Sending tor SIGHUP signal.")
    time.sleep(60)

# {'person_id': '119847', 'areas_of_practice': ['Family - Advanced Accredited'], 'employer_address': ['Dutton Gregory LLP', 'Ambassador House', '8 Carlton Crescent', 'Southampton', 'SO15 2EY'], 'solicitor_id': '468235', 'accreditations': ['Family - Advanced accreditation'], 'roles': ['Associate', 'Assistant'], 'languages': ['English'], 'admitted_date': '15/02/94', 'dx_address': '49653 SOUTHAMPTON 2', 'sra_id': '162857', 'type': 'SRA Regulated', 'tel': '02380 221344', 'name': 'Karen Lorraine Andrews'}


def output_person_details(url, office_dict):
    """ Returns person details. """
    person_tree = make_simple_tree(url)
    if error_page(person_tree):
        print("output_person: error_page: Error page found.")
    elif polipo_error(person_tree):
        print("Page not in polipo cache. Writing to missing_pages.txt")
        write_file(missing_page_fo, url)
        polipo_on()
        output_person_details(url)
        polipo_off()
    elif busy_page(person_tree):
        kill_tor()
        time.sleep(60)
        print("Waiting 60 seconds and trying again.")
        output_person_details(url)
    else:
        output = {}
        Name = office_name(person_tree)
        Office_name = person_office_name(person_tree)
        Work_address = office_dict('address')
        Work_location = office_dict('location')
        Role = str(person_role(person_tree))
        Admitted = str(person_admission_date(person_tree))
        Email = str(person_email_address(person_tree))
        Tel = str(person_telephone_number(person_tree))
        write_file(person_email_fo, Email)
        sraID = str(person_sraID(person_tree))
        Solicitor_ID = str(person_solicitorID(person_tree))
        Person_DX_address = str(person_DX_address(person_tree))
        Person_ID = str(person_id(url))
        Accreditations = str(person_accreditations(person_tree))
        Areas_of_Practice = str(person_areas_of_practice(person_tree))
        Languages = str(person_languages(person_tree))
        print("+++ Solicitor Name: " + Name)
        print("+++ Admitted: " + Admitted)
        print("+++ sraID: " + sraID)
        print("+++ Office Name: " + Office_name)
        print("+++ Person ID: " + Person_ID)
        print("+++ Solicitor ID: " + Solicitor_ID)
        print("+++ DX Address: " + Person_DX_address)
        print("+++ Role: " + Role)
        print("+++ Email: " + Email)
        print("+++ Tel: " + Tel)
        print("+++ Accreditations: " + Accreditations)
        print("+++ Areas of Practice: " + Areas_of_Practice)
        print("+++ Languages: " + Languages)
        person_dict = {}
        person_dict['sra_id'] = sraID
        person_dict['type'] = 'SRA Regulated'
        person_dict['name'] = Name
        person_dict['person_id'] = Person_ID
        person_dict['solicitor_id'] = Solicitor_ID
        person_dict['work_office'] = Office_name
        person_dict['address'] = Work_address
        person_dict['location'] = Work_location
        person_dict['admitted_date'] = Admitted
        person_dict['dx_address'] = Person_DX_address
        person_dict['tel'] = Tel
        person_dict['email'] = Email
        person_dict['accreditations'] = Accreditations
        person_dict['areas_of_practice'] = Areas_of_Practice
        person_dict['languages'] = Languages
        person_dict['roles'] = Role
#        with open('json_out/person/person_' + person_dict['person_id'] + '.json', 'w') as outfile:
#            json.dump(person_dict, outfile)
        filename = 'json_out/person/person_' + person_dict['person_id'] + '.json'
        content = json.dumps(person_dict, sort_keys=True, indent=4)
        write_json(filename, content)

# {'sra_id': '496960', 'email': 'contact@duttongregory.co.uk', 'type': 'Recognised body law practice', 'web': 'http://duttongregory.co.uk', 'dx_address': 'DX 2515 WINCHESTER', 'address': ['Trussell House,', '23 St. Peter Street,', 'Winchester,', 'Hampshire,', 'SO23 8BT,', 'England'], 'lat': '-1.313652', 'name': 'Dutton Gregory LLP', 'facialities': ['Office does not provide Sign Language', 'Office has no disabled access', 'Office does not support hearing induction loops', 'Office accepts legal aid'], 'solicitor_id': '468235', 'lon': '51.06269', 'tel': '01962 844333'}


def output_office_details(url, results_tree, idnumber):
    """ Returns office details - results_tree and idnumber only needed for lat lon """
    office_tree = make_simple_tree(url)
    if error_page(office_tree):
        print("output_office: error_page: Error page found.")
    elif polipo_error(office_tree):
        print("Page not in polipo cache. Writing to missing_pages.txt")
        write_file(missing_page_fo, url)
        polipo_on()
        output_office_details(url, results_tree, idnumber)
        polipo_off()
    elif busy_page(office_tree):
        kill_tor()
        time.sleep(60)
        print("Waitng 60 seconds and trying again.")
        output_office_details(url, results_tree, idnumber)
    else:
        Name = str(office_name(office_tree))
        Telephone = str(office_telephone_number(office_tree))
        Website = str(office_website_url(office_tree))
        write_file(firm_url_fo, Website)
        Email = str(office_email_address(office_tree))
        write_file(office_email_fo, Email)
        Lon = str(office_longitude(results_tree, idnumber))
        Lat = str(office_latitude(results_tree, idnumber))
        LatLon = {}
        LatLon['lat'] = Lat
        LatLon['lon'] = Lon
        sraID = str(office_sraID(office_tree))
        Solicitor_ID = str(office_id(url))
        Solicitor_type = str(office_solicitor_type(office_tree))
        Address = office_address(office_tree)
        people_url = str(office_people_url(office_tree))
        DX_Address = str(office_DX_address(office_tree))
        Facilities = str(office_facilities(office_tree))
        print("-- Company Name: " + Name)
        print("-- Telephone Number: " + Telephone)
        print("-- Website URL: " + Website)
        print("-- Email: " + Email)
        print("-- sraID: " + sraID)
        print("-- Solicitor ID: " + Solicitor_ID)
        print("-- Long: " + Lon)
        print("-- Lat: " + Lat)
        print("-- Solicitor Type: " + Solicitor_type)
        print("-- Address: ")
        print(Address)
        print("-- Tel: " + Telephone)
        print("-- DX Address: " + DX_Address)
        print(type(Facilities))
        print("-- Facilities: " + Facilities)
    parsed_uri = urlparse(Website)
    domain = '{uri.scheme}://{uri.netloc}/'.format(uri=parsed_uri)
    geo_content = (domain + ' ' + Lat + ' ' + Lon)
    write_file(geodata_fo, geo_content)
    office_dict = {}
    office_dict['sra_id'] = sraID
    office_dict['solicitor_id'] = Solicitor_ID
    office_dict['name'] = Name
    office_dict['email'] = Email
    office_dict['web'] = Website
    office_dict['dx_address'] = DX_Address
    office_dict['location'] = LatLon
#    office_dict['lat'] = Lat
#    office_dict['lon'] = Lon
    office_dict['tel'] = Telephone
    office_dict['type'] = Solicitor_type
    office_dict['address'] = Address
    office_dict['facilities'] = Facilities
#        with open('json_out/office/office_' + office_dict['solicitor_id'] + '.json', 'w') as outfile:
#            json.dump(office_dict, outfile)
    filename = 'json_out/office/office_' + office_dict['solicitor_id'] + '.json'
    content = json.dumps(office_dict, sort_keys=True, indent=4)
    write_json(filename, content)
    person_loop(people_url, office_dict)


def read_file(filename, url):
    """ Reads a string from a file. """
    fo = open(filename, "r+")
    str = fo.read(10)
    fo.close()


# def write_file(filename, content):
#   """ Wtites a string to a file. """
#    # Open a file
#    fo = open(filename, "a")
#    fo.write( content );
#    fo.close()


# only write if not present in file. Odd syntax but I've see it before,  must look up list,
# set and dictionary comprehensions. This seems not to work as intended if a+ is used but does with r.
# so fo.seek() is needed to begin search from the beginning rather than the append position.
def write_file(filename, content):
    """ Writes content to a file. """
    with open(filename, "a+") as fo:
        fo.seek(0)
        if not any(content == x.rstrip('\r\n') for x in fo):
            fo.write(content + '\n')
#            fo.close()
            print("write_file: content not present, writing.")
        else:
            print("write_file: content present, next.")


def write_json(filename, content):
    """ Writes json to a file. """
    with open(filename, "w") as fo:
        fo.write(content)
#        fo.close()


def write_prefix(filename, content):
    """ Writes the prefix to a file. """
    if len(content) > 0:
        # Open a file
        with open(filename, "w") as fo:
            fo.write(content)
#           fo.close()


def read_prefix(filename):
    """ Reads the prefix from a file. """
    with open(filename, 'r') as fo:
        first_line = fo.readline()
        return first_line


def person_loop(url, office_dict):
    """ The main person loop. """
    print("person_loop: url: " + str(url))
    results_url = url
    # person_results_xpath = "string(.//*[@id='main-content']/div/div[1]/div[2]/h1/strong[1])"
    person_results_xpath = ".//*[@class='search-results-controls']//h1/strong/text()"
    results_tree = make_simple_tree(results_url)
    results = apply_xpath(results_tree, person_results_xpath)
    print("person_loop: results: str(results[0]): " + str(results[0]))
    print("person_loop: results: results[0] type: " + str(type(results[0])))
    # results = int(results.replace(',',''))
    results = int(results[0])

    print("Number of Persons in office: " + str(results))
    for i in range(int(math.ceil(results/20.0))):
        page_urls = person_url(results_tree)
        # print(page_urls0
        # print(type(page_urls))
        for url in page_urls:
            print("person_loop: person_url: page_urls: " + url)
            # idnumber = people_office_id(url)
            url = office_make_absolute(url)
            with open(person_url_fo, 'r') as fo:
                if not any(url == x.rstrip('\r\n') for x in fo):
                    print("New URL, writing to file." + " - " + url)
                    write_file(person_url_fo, url)
                    output_person_details(url)
                else:
                    print("URL already fetched, next.")
                    output_person_details(url, office_dict)


def office_loop(results, letter):
    """ The main office loop. """
    for i in range(int(math.ceil(results/20.0))):
        results_tree = make_results_tree(letter, i+1)
        # Get all of the office link urls
        page_urls = office_url(results_tree)
        # print("Main: Type page_urls: " + str(type(page_urls)))
        # print("Main: page_urls: " + str(page_urls))
        for url in page_urls:
            idnumber = office_id(url)
            url = office_make_absolute(url)
            with open(office_url_fo, 'r') as fo:
                if not any(url == x.rstrip('\r\n') for x in fo):
                    print("New URL, writing to file." + " - " + url)
                    write_file(office_url_fo, url)
                    output_office_details(url, results_tree, idnumber)
                else:
                    print("URL already fetched, next.")
                    output_office_details(url, results_tree, idnumber)


headers = {
    "Host": "127.0.0.1",
    "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv':' 28.0) Gecko / 20100101 Firefox / 28.0",
    "Accept": "text/html, application/xhtml + xml, application / xml; q = 0.9, * / *; q = 0.8",
    "Accept-Language": "en-US, en; q = 0.5",
    "Accept-Encoding": "gzip, deflate",
    "Referer": "http://127.0.0.1/drupal-7.31/",
    "Cookie": "Drupal.toolbar.collapsed = 0; Drupal.tableDrag.showWeight = 0; has_js = 1",
    "Connection": "keep-alive",
    "Content-Type": "application / x-www-form-urlencoded",
    "Content-Length": " 231"
}


# def main(name, max_results):
#    """ Creates squence letters for the main loops. """
#    allTheLetters_list = list(string.lowercase)
#    for letter in allTheLetters_list:
#        # how many beinging with letter?
#        results = get_numberOfResults(letter, 1)
#        if results <= max_results:
#            print("--------------------------------------------------------------------------------")
#            print("Single letter results: " + str(results) )
#            print("--------------------------------------------------------------------------------")
#            office_loop(results, letter)
#        else:
#            for letter2 in allTheLetters_list:
#                # letter = letter + allTheLetters_list[allTheLetters_list_index]
#                prefix = letter + letter2
#                # allTheLetters_list_index += 1
#                print(prefix)
#                results = get_numberOfResults(prefix, 1)
#                # how many beinging with prefix?
#                if results <= max_results:
#                    print("--------------------------------------------------------------------------------")
#                    print("Double letter results: " + str(results))
#                    print("--------------------------------------------------------------------------------")
#                    office_loop(results, prefix)
#                else:
#                    for letter3 in allTheLetters_list:
#                        # letter = letter + allTheLetters_list[allTheLetters_list_index]
#                        prefix = letter + letter2 + letter3
#                        # allTheLetters_list_index += 1
#                        print(prefix)
#                        results = get_numberOfResults(prefix, 1)
#                        if results <= max_results:
#                            print("--------------------------------------------------------------------------------")
#                            print("Triple letter results: " + str(results) )
#                            print("--------------------------------------------------------------------------------")
#                            office_loop(results, prefix)
#                        else:
#                            print("We need more letters in MAIN")

# ------------------------
'''
The lawsoc search result page limits to 500 returned results over several pages.

Just searching for single letters returns more than 500 results for some letters.

The main deals with this by identifying when more than 500 results get returned and then adding an additional letter to the search until the number of results is within the results limit. Currently only three letters are implemented.

Additionally to prevent starting from the beginning again, in the event the crawl fails, it starts from the position recorded in lawsoc_prefix.txt

1. Pass in the max search results(500) and the last position(name) from lawsoc_prefix.txt
2. Create a list of the alphabet to index against.
3. Set the initial index: the index of the first letter of 'name' in the alphabet list or 0 if its empty.
4. the first 'for' loop:
    looping through all the letters from the initial index to the end of the alphabet list:
    get the number of results and if over the limit go to the next letter for loop
    if its within the limit go into the office loop to extract data
    and write the current prefix to the lawsoc_prefix.txt
5. before the second 'for' loop we set the initial index for the second letter to begin from the stored position
6. the second 'for' loop: pretty much as before
'''


def main(max_results, name='a'):
    """ Main loop to call the office and person loops. Submitting letter sequences to the search page. """
    allTheLetters_list = list(string.ascii_lowercase)
    print("main: name: " + name)
    first_pass = True
    if len(name) >= 1 and first_pass:
        name_idx = allTheLetters_list.index(name[0])
    else:
        name_idx = 0

    for letter in allTheLetters_list[name_idx:]:
        # how many beginning with letter?
        results = get_numberOfResults(letter, 1)
        if results <= max_results:
            print("--------------------------------------------------------------------------------")
            print("Single letter results: " + str(results))
            print("--------------------------------------------------------------------------------")
            office_loop(results, letter)
        else:
            print("--------------------------------------------------------------------------------")
            print("ERROR Single: max_results hit.")
            print("--------------------------------------------------------------------------------")

        if first_pass and len(name.strip('\n')) >= 2:
            name_idx2 = allTheLetters_list.index(name[1])
        else:
            name_idx2 = 0

        for letter2 in allTheLetters_list[name_idx2:]:
            # letter = letter + allTheLetters_list[allTheLetters_list_index]
            prefix = letter + letter2
            # allTheLetters_list_index += 1
            print(prefix)
            results = get_numberOfResults(prefix, 1)
            # how many begining with prefix?
            if results <= max_results:
                print("--------------------------------------------------------------------------------")
                print("Double letter results: " + str(results))
                print("--------------------------------------------------------------------------------")
                office_loop(results, prefix)
            else:
                print("--------------------------------------------------------------------------------")
                print("ERROR Double: max_results hit.")
                print("--------------------------------------------------------------------------------")

            if first_pass and len(name.strip('\n')) >= 3:
                name_idx3 = allTheLetters_list.index(name[2])
            else:
                name_idx3 = 0

            for letter3 in allTheLetters_list[name_idx3:]:
                # letter = letter + allTheLetters_list[allTheLetters_list_index]
                prefix = letter + letter2 + letter3
                write_prefix(prefix_fo, prefix)
                # allTheLetters_list_index += 1
                print(prefix)
                results = get_numberOfResults(prefix, 1)
                if results <= max_results:
                    print("--------------------------------------------------------------------------------")
                    print("Triple letter results: " + str(results))
                    print("--------------------------------------------------------------------------------")
                    office_loop(results, prefix)
                else:
                    print("--------------------------------------------------------------------------------")
                    print("ERROR Triple: max_results hit.")
                    print("--------------------------------------------------------------------------------")
            first_pass = False

#                if first_pass and len(name) >= 4:
#                    name_idx4 = allTheLetters_list.index(name[3])
#                else:
#                    name_idx4 = '0'
#                for letter4 in allTheLetters_list[name_idx4:]:
#                    # letter = letter + allTheLetters_list[allTheLetters_list_index]
#                    prefix = letter + letter2 + letter3 + letter4
#                    # allTheLetters_list_index += 1
#                    print(prefix)
#                    results = get_numberOfResults(prefix, 1)
#                    if results <= max_results:
#                        print("--------------------------------------------------------------------------------")
#                        print("Quad letter results: " + str(results) )
#                        print("--------------------------------------------------------------------------------")
#                        write_prefix(prefix_fo, prefix)
#                        office_loop(results, prefix)
#                    else:
#                        print("--------------------------------------------------------------------------------")
#                        print("Error Quad: max_results hit.")
#                        print("--------------------------------------------------------------------------------")
#               first_pass = False


# the maximum allowed by lawsoc per search
max_results = 500

# set the output filenames
fo_home = os.getcwd()
lawsoc_results_fo = os.path.join(fo_home, "lawsoc_results_urls.txt")
missing_page_fo = os.path.join(fo_home, "missing_files.txt")
office_url_fo = os.path.join(fo_home, "lawsoc_office_urls.txt")
office_email_fo = os.path.join(fo_home, "lawsoc_office_email.txt")
person_url_fo = os.path.join(fo_home, "lawsoc_person_urls.txt")
person_email_fo = os.path.join(fo_home, "lawsoc_person_email.txt")
firm_url_fo = os.path.join(fo_home, "lawsoc_firm_urls.txt")
prefix_fo = os.path.join(fo_home, "lawsoc_prefix.txt")
geodata_fo = os.path.join(fo_home, "geodata.txt")

# the maximum allowed by lawsoc per search
max_results = 500

# socks proxy used by tor and the tor instance control port (hashed password generated by 'tor --hash-password "insonik"' and put in torrc:
# 'HashedControlPassword 16:23B2A5919010B21660B564BCA0ECB94A2F2BB5373708D709BF44CC8CCE'
# "ControlPort 9151
socks_proxy = '9150'
controlport = '9151'
torctl_password = 'insonik'

# read in letters processed so far, to be passed to main to continue from where we left off.
# if its blank default to 'a'
if len(read_prefix(prefix_fo)) > 0:
    name = read_prefix(prefix_fo)
    print("type name: " + str(type(name)))
    print("Beginning from: " + str(name))
else:
    name = 'a'
    print("Prefix file empty, beginning from: " + str(name))

if __name__ == "__main__":
    main(max_results, name)
