#!/usr/bin/python3

# Import system modules...
import sys,os,time,logging,json
from pyelasticsearch import ElasticSearch
from pyelasticsearch import bulk_chunks

office_mapping = {"mappings": {
                "office": {
                    "properties" : {
                        "location": {
                            "type": "geo_point"
                            },
                        "solicitor_id": {
                            "type": "string"
                            },
                        "sra_id": {
                            "type": "string"
                            },
                        "dx_address": {
                            "type": "string"
                            },
                        "email": {
                            "type": "string"
                            },
                        "facilities": {
                            "type": "string"
                            },
                        "name": {
                                "type": "string"
                                },
                        "tel": {
                                "type": "string"
                                },
                        "web": {
                                "type": "string"
                                },
                        "address": {
                                "type": "string"
                                }
                        }
                    }
                } }

person_mapping = {"mappings": {
                "person": {
                    "properties" : {
                        "admitted_date": {
                            "type": "date",
                            "format": "yyyy/MM/dd",
                            "ignore_missing" : True
                            },
                        "person_id": {
                            "type": "integer"
                            },
                        "solicitor_id": {
                            "type": "integer"
                            },
                        "sra_id": {
                            "type": "integer"
                            },
                        "type": {
                            "type": "string"
                            },
                        "roles": {
                            "type": "string"
                            },
			"name":{
                                "type": "string"
                                },
                        "languages": {
                                "type": "string"
                                },
                        "email": {
                                "type": "string"
                                },
                        "dx_address": {
                                "type": "string"
                                },
                        "areas_of_practice": {
                                "type": "string"
                                },
                        "accreditations": {
                                "type": "string"
                                },
                        "tel": {
                                "type": "string"
                                }
                        } }
                        } }

def create_lawsoc_index(client, index):
    # create an empty index
    client.indices.create(
        index=index,
        body={
          'settings': {
            # just one shard, no replicas for testing
            'number_of_shards': 1,
            'number_of_replicas': 0,
            "analysis" : {
                "filter" : {
                    "email" : {
                        "type" : "pattern_capture",
			"preserve_original" : 1,
			"patterns" : [
                            "([^@]+)",
                            "(\\p{L}+)",
                            "(\\d+)",
                            "@(.+)"
                            ]
		      },
		      "phonetic_metaphone" : {
                          "encoder" : "metaphone",
			  "replace" : false,
			  "type" : "phonetic"
		      },
		      "default_delimiter" : {
                          "type": "word_delimiter",
			  "generate_word_parts": true,
			  "generate_number_parts": true,
			  "catenate_words": false,
			  "catenate_numbers": false,
			  "catenate_all": false,
			  "split_on_case_change": true,
			  "preserve_original": false,
			  "split_on_numerics": true,
			  "stem_english_possessive": true
		      },
		      "my_delimiter" : {
                          "type": "word_delimiter",
			  "generate_word_parts": true,
			  "generate_number_parts": true,
			  "catenate_words": false,
			  "catenate_numbers": false,
			  "catenate_all": false,
			  "split_on_case_change": false,
			  "preserve_original": true,
			  "split_on_numerics": false,
			  "stem_english_possessive": false
		      }
	      },
	      "analyzer" : {
                      "index_analyzer" : {
                          "tokenizer" : "standard",
			  "filter" : [
                              "standard",
                              "lowercase",
                              "stop",
                              "asciifolding",
                              "phonetic_metaphone",
                              "my_delimiter"
                              ]
		      },
		      "search_analyzer" : {
                          "tokenizer" : "standard",
			  "filter" : [
                              "standard",
                              "lowercase",
                              "stop",
                              "asciifolding",
                              "phonetic_metaphone"
                              ]
		      },
		      "name_analyzer" : {
                          "type" : "custom",
                          "tokenizer" : "standard",
                          "filter" : [
                              "asciifolding",
                              "lowercase"
                              ]
		      },
   	              "email_analyzer" : {
                          "type" : "custom",
			  "tokenizer" : "uax_url_email",
			  "filter" : [
                              "email",
                              "lowercase",
                              "unique"
                              ]
		      },
   	              "url_analyzer" : {
                          "type" : "custom",
                          "tokenizer" : "uax_url_email"
		      },
		      "metaphone_analyzer" : {
                          "type" : "custom",
			   "tokenizer" : "standard",
			   "filter" : [
                               "phonetic_metaphone"
                               ]
		      }
              }
            }
           }
        },
        # ignore already existing index
        ignore=409
        )

def load_lawsoc(client, path=None, index='lawsoc', document_type='office', document_id=1):
    # create the parent document in case it doesn't exist
    client.create(
        index=index,
        doc_type=document_type,
        id=document_id,
        body={},
        ignore=409 # 409 - conflict - would be returned if the document is already there
    )

    create_lawsoc_index(client, index)

def documents(workspace, fileList):
    print("Entering documents function")
    print("length of fileList: " + str(len(fileList)))
    #for file in fileList:
    for index, file in enumerate(fileList):
        print(index, '--->', file)
        # For each file in the list, read its json and provide a generator
        contents = json.load(open(workspace + file))
        yield es.index_op(contents)


# set some variables
# Set the workspace...
# workspace_home = '/var/tmp/lawsoc_new/'
workspace_home = os.getcwd()
workspace = workspace_home + "/json_out/"
office_workspace = workspace + "office/"
person_workspace = workspace + "person/"
log_workspace = workspace + "es_trace.log"

office_document_type = "office_detail"
person_document_type = "person_detail"

index_date = time.strftime("%Y-%m-%d")
es_index_name = "lawsoc_" + index_date

if __name__ == '__main__':
    #logging.basicConfig()
    #tracer = logging.getLogger('elasticsearch.trace').setLevel(logging.INFO)
    #tracer.setLevel(logging.INFO)
    #tracer.addHandler(logging.FileHandler(log_workspace))

    # Create the ES connection object...
    es = ElasticSearch('http://localhost:9200/')

    # Get a list of files in the workspace...
    office_fileList = os.listdir(office_workspace)
    person_fileList = os.listdir(person_workspace)

    # Run through the office loop...
    try:
        for chunk in bulk_chunks(documents(office_workspace, office_fileList),
            docs_per_chunk=100,
            bytes_per_chunk=10000
            ):
            es.bulk(chunk, doc_type=office_document_type, index=es_index_name)
    except Exception as e:
        print(e)

    print("Finished office json files.")
    time.sleep(5)

    # Run through the person loop...
    try:
        for chunk in bulk_chunks(documents(person_workspace, person_fileList),
            docs_per_chunk=100,
            bytes_per_chunk=10000
            ):
            es.bulk(chunk, doc_type=person_document_type, index=es_index_name)
    except Exception as e:
        print(e)
