# -*- coding: utf-8 -*-
# The contents of this file are subject to the Mozilla Public
# License Version 1.1 (the "License"); you may not use this file
# except in compliance with the License. You may obtain a copy of
# the License at http://www.mozilla.org/MPL/
#
# Software distributed under the License is distributed on an "AS
# IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
# implied. See the License for the specific language governing
# rights and limitations under the License.
#
# The Original Code is Reportek version 1.0.
#
# The Initial Developer of the Original Code is European Environment
# Agency (EEA).  Portions created by Finsiel are
# Copyright (C) European Environment Agency.  All
# Rights Reserved.
#
# Contributor(s):
# Soren Roug, EEA
# Miruna Badescu, Eau de Web
# Cristian Romanescu, Eau de Web


"""EnvelopeCustomDataflows

This class which Envelope subclasses from contains functions specific to one or more dataflows.
When writing in this class, specify the name of the dataflow as comment first

"""

# Zope imports
from Globals import DTMLFile, MessageDialog, InitializeClass
from Products.PageTemplates.PageTemplateFile import PageTemplateFile
try: from zope.contenttype import guess_content_type # Zope 2.10 and newer
except: from zope.app.content_types import guess_content_type # Zope 2.9 and older
from AccessControl import ClassSecurityInfo
from DateTime import DateTime
import xmlrpclib
import string
import logging
import transaction
from xml.dom.minidom import parseString

# Product specific imports
import RepUtils
from constants import WEBQ_XML_REPOSITORY, CONVERTERS_ID
from zip_content import ZZipFile
from XMLInfoParser import detect_single_schema
from constants import QAREPOSITORY_ID
import zip_content
from DataflowsManager import DataflowsManager

from os.path import join
import tempfile
import os, traceback
from zipfile import *

conversion_log = logging.getLogger(__name__ + '.conversion')

# Article 17 constants
#GENERAL_SCHEMA = 'http://biodiversity.eionet.europa.eu/schemas/dir9243eec/generalreport.xsd'
#HABITAT_SCHEMA = 'http://biodiversity.eionet.europa.eu/schemas/dir9243eec/habitats.xsd'
#SPECIE_SCHEMA = 'http://biodiversity.eionet.europa.eu/schemas/dir9243eec/species.xsd'
#GML_SCHEMA = 'http://biodiversity.eionet.europa.eu/schemas/dir9243eec/gml_art17.xsd'
ESRI_EXTENSIONS = ['shp', 'shx', 'dbf', 'prj', 'xml']
SHORT_ESRI_EXTENSIONS = ['shp', 'shx', 'dbf', 'prj']
EXTENDED_ESRI_EXTENSIONS = ['shp', 'shx', 'dbf', 'prj', 'xml', 'shp.xml']
OPTIONAL_ESRI_EXTENSIONS = ['xml', 'shp.xml']
ESRI_EXTRAEXTENSIONS = ['.shp', '.shx', '.dbf', '.prj', '.xml']


def invoke_conversion_service(server_name, method_name, url):
    server = xmlrpclib.ServerProxy(server_name)
    method = getattr(server.ConversionService, method_name)
    return method(url, '')


class EnvelopeCustomDataflows:
    """ This class which Envelope subclasses from contains functions specific to one or more dataflows.
    """

    security = ClassSecurityInfo()

    ##################################################
    #   Generic methods for all dataflows
    ##################################################

    #generic method that converts Excel files (XLS or ODS) to XML
    #if the Excel template comes from Data Dictionary
    security.declareProtected('Change Envelopes', 'upload_excel_file')
    upload_excel_file = DTMLFile('dataflows/envelope_upload_excel_file',globals())

    #generic method that converts Excel files (XLS or ODS) to XML
    security.declareProtected('Change Envelopes', 'upload_generic_excel_file')
    upload_generic_excel_file = DTMLFile('dataflows/envelope_upload_generic_excel_file',globals())

    #generic method that uploads a DD file or accompanying data
    security.declareProtected('Change Envelopes', 'upload_dd_file')
    upload_dd_file = DTMLFile('dataflows/envelope_upload_dd_file',globals())

    #generic method that uploads a single file or a zip
    security.declareProtected('Change Envelopes', 'upload_doc_or_zip')
    upload_doc_or_zip = PageTemplateFile('zpt/envelope/upload_doc_or_zip', globals())

    def _get_xml_files_by_schema(self, schema):
        """ Returns the list of XML files with the given schema from that envelope """
        return [doc.id for doc in self.objectValues('Report Document') if doc.xml_schema_location == schema]

    security.declareProtected('Change Envelopes', 'convert_excel_file')
    def convert_excel_file(self, file, restricted='', strict_check=0, conversion_function='', REQUEST=None):
        """ Uploads the original spreadsheet to the envelope
            Attempts the conversion of the DD-based spreadsheet
            If successful, deletes all XML files in the envelope previously generated by that file
            Adds the converted XML files to the envelope
            Returns the error message on REQUEST and the result code if no REQUEST is given:

            -   1 if the conversion succeeded, with or without validation errors
            -   0 if the conversion did not succeed and the original file was uploaded
            -   -1 if no upload has been done

        """
        if not file or type(file) is type('') or not hasattr(file,'filename'):
            if REQUEST is not None:
                return self.messageDialog(
                                message='Upload failed! No file was specified!',
                                action='index_html')
            else:
                return -1
        l_original_content = file.read()
        # build original file id
        l_id = file.filename
        if l_id:
            l_id = l_id[max(string.rfind(l_id,'/'), string.rfind(l_id,'\\'), string.rfind(l_id,':'))+1:]
            l_id = l_id.strip()
            l_id = RepUtils.cleanup_id(l_id)

        # delete previous version of file if exists
        if hasattr(self, l_id):
            self.manage_delObjects(l_id)
        if l_id.lower().endswith('.xls') or l_id.lower().endswith('.xlsx'):
            l_original_type = 'Excel file'
        elif l_id.endswith('.ods'):
            l_original_type = 'Spreadsheet file'
        else:
            l_original_type = 'Data file'
        # upload original file in the envelope
        self.manage_addDocument(id=l_id, title=l_original_type, file=l_original_content, restricted=restricted)
        if strict_check and l_original_type == 'Data file':
            if REQUEST is not None:
                return self.messageDialog(
                                message='The file was successfully uploaded in the envelope.',
                                action='index_html')
            else:
                return 0

        # must commit transaction first, otherwise the file is not accessible from outside
        transaction.commit()
        l_doc = getattr(self, l_id)

        try:
            # XML/RPC call to the converter
            l_server_name = getattr(self, CONVERTERS_ID).remote_converter
            l_url = self.absolute_url() + '/' + l_id
            l_ret_list = invoke_conversion_service(l_server_name, 'convertDD_XML_split', l_url)

            # the result is a dictionary with the following elements:
            #   resultCode (String): 0 – success; 1- converted with validation errors; 2- system error; 3 – schema not found or expired error
            #   resultDescription (String): short textual description about conversion results. If resultCode > 0, then resultDescription contains error message
            #   conversionLog (String): conversion log in HTML format. The result is HTML fragment wrapped into <div class="feedback"> element.
            #   convertedFiles (Array<Struct>): list of dictionaries of converted files
            #       fileName (String): Name of the result file
            #       content (Byte[]): XML content of result document as a UTF-8 encoded byte array

            # delete the XML files that may be previously generated by conversion of the same file
            self.manage_delObjects([x.id for x in self.objectValues('Report Document') if x.content_type == 'text/xml' and x.id.startswith(l_id[:-4] + '_')])
            # delete the feedback that may be previously generated by conversion of the same file
            if hasattr(self, 'conversion_log_%s' % l_id):
                self.manage_delObjects('conversion_log_%s' % l_id)

            l_result = l_ret_list['resultCode']
            if l_result in ['0', '1']:
                # add XML files
                l_converted_files = l_ret_list['convertedFiles']
                for l_xml in l_converted_files:
                    l_xml_id = l_id[:-4] + '_' + l_xml['fileName']
                    self.manage_addDocument(id=l_xml_id, title='Converted from - %s' % l_id, file=l_xml['content'].data, content_type='text/xml', restricted=restricted)
                # Change the original file title to show the conversion result
                l_doc.manage_editDocument(title='%s - converted into an XML delivery' % l_original_type, content_type=l_doc.content_type)
                # add feedback with the conversion log
                self.manage_addFeedback(id='conversion_log_%s' % l_id, 
                        title='Conversion log for file %s' % l_id, 
                        feedbacktext=l_ret_list['conversionLog'], 
                        automatic=1, 
                        content_type='text/html',
                        document_id=l_id)
                if REQUEST is not None:
                    if len(l_converted_files) == 0:
                        l_msg = 'The file was successfully uploaded in the envelope, but not converted into an XML delivery because the file contains no data.'
                    elif l_result == '1':
                        l_msg = 'The file was successfully uploaded in the envelope and converted into an XML delivery. The conversion contains validation warnings - see the Feedback posted for this file for details.'
                    else:
                        l_msg = 'The file was successfully uploaded in the envelope and converted into an XML delivery.'
                    return self.messageDialog(
                                message=l_msg,
                                action='index_html')
                else:
                    return 1
            elif l_result == '2':
                # Change the original file title to show the conversion result
                l_doc.manage_editDocument(title='%s - not converted into an XML delivery' % l_original_type, content_type=l_doc.content_type)
                # add feedback with the conversion log
                self.manage_addFeedback(id='conversion_log_%s' % l_id, 
                        title='Conversion log for file %s' % l_id, 
                        feedbacktext=l_ret_list['conversionLog'], 
                        automatic=1, 
                        content_type='text/html',
                        document_id=l_id)

                if REQUEST is not None:
                    conversion_log.error(
                            l_ret_list.get('resultDescription',
                                           'Error in converting file at %s' %l_doc.absolute_url())
                    )
                    return self.messageDialog(
                                    message='The file was successfully uploaded in the envelope, but not converted into an XML delivery. See the Feedback posted for this file for details.',
                                    action='index_html')
                else:
                    return 0

            elif l_result == '3':
                # Change the original file title to show the conversion result
                l_doc.manage_editDocument(title='%s - not converted into an XML delivery - not based on the most recent reporting template' % l_original_type, content_type=l_doc.content_type)
                # add feedback with the conversion log
                self.manage_addFeedback(id='conversion_log_%s' % l_id, 
                        title='Conversion log for file %s' % l_id, 
                        feedbacktext=l_ret_list['conversionLog'], 
                        automatic=1, 
                        content_type='text/html',
                        document_id=l_id)
                if REQUEST is not None:
                    return self.messageDialog(
                                    message='The file was successfully uploaded in the envelope, but not converted into an XML delivery, because you are not using the most recent reporting template - %s. See the feedback posted for this file for details.' % l_ret_list['resultDescription'],
                                    action='index_html')
                else:
                    return 0
            else:
                if REQUEST is not None:
                    return self.messageDialog(
                                    message='Incorrect result from the Conversion service. The file was successfully uploaded in the envelope, but not converted into an XML delivery.',
                                    action='index_html')
                else:
                    return 0

        except Exception, err:
            # Change the original file title to show the error in conversion
            l_doc.manage_editDocument(title='%s, could not be converted into an XML delivery' % l_original_type, content_type=l_doc.content_type)
            if REQUEST is not None:
                conversion_log.error(
                        l_ret_list.get('resultDescription',
                                       'Error in converting file at %s' %l_doc.absolute_url())
                )
                return self.messageDialog(
                                message='The file was successfully uploaded in the envelope, but not converted into an XML delivery because of a system error: %s' % str(err),
                                action='index_html')
            else:
                return 0

    security.declareProtected('Change Envelopes', 'convert_generic_excel_file')
    def convert_generic_excel_file(self, file, restricted='', strict_check=0, REQUEST=None):
        """ Uploads the original spreadsheet to the envelope
            Attempts the conversion of the spreadsheet
            If successful, deletes all XML files in the envelope previously generated by that file
            Adds the converted XML files to the envelope
            Returns the error message on REQUEST and the result code if no REQUEST is given:
            -   1 if the conversion succeeded
            -   0 if the conversion did not succeed and the original file was uploaded
            -   -1 if no upload has been done
        """
        if not file or type(file) is type('') or not hasattr(file,'filename'):
            if REQUEST is not None:
                return self.messageDialog(
                                message='Upload failed! No file was specified!',
                                action='index_html')
            else:
                return -1
        l_original_content = file.read()
        # build original file id
        l_id = file.filename
        if l_id:
            l_id = l_id[max(string.rfind(l_id,'/'), string.rfind(l_id,'\\'), string.rfind(l_id,':'))+1:]
            l_id = l_id.strip()
            l_id = RepUtils.cleanup_id(l_id)

        # delete previous version of file if exists
        if hasattr(self, l_id):
            self.manage_delObjects(l_id)
        if l_id.endswith('.xls') or l_id.endswith('.xlsx'):
            l_original_type = 'Excel file'
        elif l_id.endswith('.ods'):
            l_original_type = 'Spreadsheet file'
        else:
            l_original_type = 'Data file'
        # upload original file in the envelope
        self.manage_addDocument(id=l_id, file=l_original_content, restricted=restricted)
        if strict_check and l_original_type == 'Data file':
            if REQUEST is not None:
                return self.messageDialog(
                                message='The file was successfully uploaded in the envelope, but not converted into an XML delivery because it is not an Excel file.',
                                action='index_html')
            else:
                return 0

        # must commit transaction first, otherwise the file is not accessible from outside
        transaction.commit()
        l_doc = getattr(self, l_id)

        try:
            # XML/RPC call to the converter
            l_server_name = getattr(self, CONVERTERS_ID).remote_converter
            l_url = self.absolute_url() + '/' + l_id
            l_server = xmlrpclib.ServerProxy(l_server_name)
            l_ret_list = l_server.ConversionService.convertExcelToXML(self.absolute_url() + '/' + l_id)

        except Exception, err:
            # Change the original file title to show the error in conversion
            l_doc.manage_editDocument(title='%s, could not be converted into an XML delivery' % l_original_type, content_type=l_doc.content_type)
            if REQUEST is not None:
                return self.messageDialog(
                                message='The file was successfully uploaded in the envelope, but not converted into an XML delivery, probably because you are not using the most recent reporting template.',
                                action='index_html')
            else:
                return 0

        l_result = []
        l_succeded = 0
        # if the conversion succeded, delete the XML files
        # that may be previously generated a conversion of the same file
        if l_ret_list[0] == '0':
            l_succeded = 1
            self.manage_delObjects([x.id for x in self.objectValues('Report Document') if x.content_type == 'text/xml'])

        # The first element is the error code:
        #    0 – STATUS_OK
        #    1 – STATUS_ERR_VALIDATION
        #    2 – STATUS_ERR_SYSTEM
        #    3 – STATUS_ERR_SCHEMA_NOT_FOUND
        if l_ret_list[0] == '0':
            if len(l_ret_list) == 2:
                # Change the original file title
                l_doc.manage_editDocument(title='%s, empty delivery' % l_original_type, content_type=l_doc.content_type)
                if REQUEST is not None:
                    return self.messageDialog(
                                    message='The file was successfully uploaded in the envelope, but not converted into an XML delivery because no data was filled in.',
                                    action='index_html')
                else:
                    return 0
            l_n = len(l_ret_list)
            l_xmls = l_ret_list[2:l_n:2]
            for i, l_xml in enumerate(l_xmls):
                l_data = l_ret_list[2*i+3]
                l_xml = l_xml.replace('.xslt', '')
                if isinstance(l_data, unicode):
                    l_data = l_data.encode('utf-8')
                self.manage_addDocument(id=l_xml, title='Converted from - %s' % l_id, file=l_data, content_type='text/xml', restricted=restricted)
            # Change the original file title to show the conversion result
            l_doc.manage_editDocument(title='%s using correct template - converted into an XML delivery' % l_original_type, content_type=l_doc.content_type)
            if REQUEST is not None:
                return self.messageDialog(
                            message='The file was successfully uploaded in the envelope and converted into an XML delivery.',
                            action='index_html')
            else:
                return 1
        elif l_ret_list[0] in ['1', '3']:
            # Change the original file title to show the conversion result
            l_doc.manage_editDocument(title='%s, not based on template for automatic processing' % l_original_type, content_type=l_doc.content_type)
            if REQUEST is not None:
                return self.messageDialog(
                                message='The file was successfully uploaded in the envelope, but not converted into an XML delivery, probably because you are not using the most recent reporting template.',
                                action='index_html')
            else:
                return 0
        else:
            l_doc.manage_editDocument(title='%s - not converted into an XML delivery' % l_original_type, content_type=l_doc.content_type)
            if REQUEST is not None:
                return self.messageDialog(
                                message='The file was successfully uploaded in the envelope, but not converted into an XML delivery due to the following error: %s' % l_ret_list[1],
                                action='index_html')
            else:
                return -1

    security.declareProtected('Change Envelopes', 'replace_dd_xml')
    def replace_dd_xml(self, file, restricted='', required_schema=[], REQUEST=None):
        """ Receives the file uploaded check that the schema id
            starts with 'http://dd.eionet.europa.eu/GetSchema?id=' or,
            if the 'required_schema' list is provided, it checks that
            the schema is one of those

            If so, it replaces the existing XML files in the envelope,
            otherwise it complains

            Returns the error message on REQUEST and the result code if no REQUEST is given:
                1 if the file was right
                0 if the file was not right
                -1 if there was no file
        """
        if not file or type(file) is type('') or not hasattr(file,'filename'):
            if REQUEST is not None:
                return self.messageDialog(
                                message='Upload failed! No file was specified!',
                                action='index_html')
            else:
                return -1
        #guess content type
        first_1k = file.read(1024)
        file.seek(0)
        content_type, enc = guess_content_type(file.filename, first_1k)
        if content_type == 'text/xml':
            #verify the XML schema
            schema = detect_single_schema(file)
            file.seek(0)
            if (not required_schema and schema.startswith('http://dd.eionet.europa.eu/GetSchema?id=')) or schema in RepUtils.utConvertToList(required_schema):
                #delete all the XML files from this envelope which containt this schema
                xmls = self._get_xml_files_by_schema(schema)
                self.manage_delObjects(xmls)

                #finally, add a Report Document
                return self.manage_addDocument(title='Data file', file=file, restricted=restricted, REQUEST=REQUEST)
            else:
                if REQUEST is not None:
                    return self.messageDialog(
                                message="The file you are trying to upload wasn't generated according to the Data Dictionary schema! File not uploaded!",
                                action='index_html')
                else:
                    return 0
        else:
            if REQUEST is not None:
                return self.messageDialog(
                                message="The file you are trying to upload is not an XML file! File not uploaded!",
                                action='index_html')
            else:
                return 0

    security.declareProtected('Change Envelopes', 'manage_addDocOrZip')
    def manage_addDocOrZip(self, file, restricted='', REQUEST=None):
        """ Adds a file or unpacks a zip in the envelope """
        if not file or type(file) is type('') or not hasattr(file,'filename'):
            if REQUEST is not None:
                return self.messageDialog(
                                message='No file was specified!',
                                action='index_html')
            return 0
        else:
            if file.filename.endswith('.zip'):
                return self.manage_addzipfile(file=file, restricted=restricted, REQUEST=REQUEST)
            else:
                return self.manage_addDocument(file=file, restricted=restricted, REQUEST=REQUEST)

    security.declareProtected('Change Envelopes', 'manage_addDDFile')
    def manage_addDDFile(self, file, restricted='', required_schema=[], REQUEST=None):
        """ Adds a DD file as follows:
            - if the file is a spreadsheet, it calls convert_excel_file
            - if the file XML, it calls replace_dd_xml
            - otherwise it calls manage_addDocOrZip
        """
        if not file or type(file) is type('') or not hasattr(file,'filename'):
            if REQUEST is not None:
                return self.messageDialog(
                                message='No file was specified!',
                                action='index_html')
            return 0
        else:
            l_filename = file.filename.lower()
            if l_filename.endswith('.xls') or l_filename.endswith('.xlsx') or l_filename.endswith('.ods'):
                return self.convert_excel_file(file=file, restricted=restricted, REQUEST=REQUEST)
            elif l_filename.endswith('.xml'):
                return self.replace_dd_xml(file=file, restricted=restricted, required_schema=required_schema, REQUEST=REQUEST)
            elif l_filename.endswith('.zip'):
                return self.manage_addzipfile(file=file, restricted=restricted, REQUEST=REQUEST)
            else:
                return self.manage_addDocument(file=file, restricted=restricted, REQUEST=REQUEST)

    security.declareProtected('View', 'subscribe_all_actors')
    def subscribe_all_actors(self, event_type=''):
        """ Calls UNS for all actors it has found in the work items in the envelope
            and subscribes them to receive notifications to a specified event if the parameter is provided
        """
        engine = self.ReportekEngine
        if engine.UNS_server:
            actors = []
            for w in self.objectValues('Workitem'):
                if w.actor != 'openflow_engine' and w.actor not in actors:
                    actors.append(w.actor)
            filters = []
            country_name = str(self.localities_dict().get(self.country, {'name':'Unknown'})['name'])
            for df in self.dataflow_uris:
                if event_type:
                    filters.append({'http://rod.eionet.europa.eu/schema.rdf#locality': country_name, 'http://rod.eionet.europa.eu/schema.rdf#obligation': engine.getDataflowTitle(df), 
                    'http://rod.eionet.europa.eu/schema.rdf#event_type': event_type})
                else:
                    filters.append({'http://rod.eionet.europa.eu/schema.rdf#locality': country_name, 'http://rod.eionet.europa.eu/schema.rdf#obligation': engine.getDataflowTitle(df)})

            engine.uns_subscribe_actors(actors, filters)

    ##################################################
    #   Conversions - SHP to GML related
    ##################################################

# replace this Article 17 specific functionality with a generic form for uploading shapefiles

#    security.declareProtected('View', 'testExistingFile')
#    def testExistingFile(self, filename):
#        """ """
#        return hasattr(self, filename)
#
#    security.declareProtected('Change Envelopes', 'convert_esri_to_xml')
#    def convert_esri_to_xml(self, file_id, shx_file, shp_file, dbf_file, xml_file, prj_file):
#        """ """
#        shp_filename = RepUtils.getFilename(shp_file.filename)
#        dbf_filename = RepUtils.getFilename(dbf_file.filename)
#        shx_filename = RepUtils.getFilename(shx_file.filename)
#        prj_filename = RepUtils.getFilename(prj_file.filename)
#        if len(shp_filename) and len(dbf_filename) and len(shx_filename) and len(prj_filename):
#
#            tmp_basename = shp_filename[:-3]
#            xml_filename = '%sxml' % tmp_basename
#
#            #pre-generation tests
#            for k in SHORT_ESRI_EXTENSIONS:
#                if eval('%s_filename' % k)[:-3] != tmp_basename:
#                    return 'All files must have the same name and different extensions.'
#            try:    prj_file_data = prj_file.read()
#            except: prj_file_data = ''
#            if len(prj_file_data) == 0:
#                return 'PRJ file is empty!'
#
#            from Products.Reportek_dependencies.GML.shp_to_gml import shp_to_gml
#            try:
#                #generate GML convertion
#                l_filename_tmp = RepUtils.cookId(shp_file)
#                l_filename = l_filename_tmp[:l_filename_tmp.rfind('.')]
#
#                gml_file_obj = getattr(self, file_id, None)
#
#                if gml_file_obj.get_accept_time():
#                    return 'Document cannot be changed since it has already accepted by the client'
#
#                #get uploaded ESRI data
#                shp_file_data = shp_file.read()
#                shx_file_data = shx_file.read()
#                dbf_file_data = dbf_file.read()
#                try:    xml_file_data = xml_file.read()
#                except: xml_file_data = ''
#
#                #temporary generate ESRI data/files
#                RepUtils.createTempFile(shp_file_data, shp_filename)
#                RepUtils.createTempFile(shx_file_data, shx_filename)
#                RepUtils.createTempFile(dbf_file_data, dbf_filename)
#                RepUtils.createTempFile(prj_file_data, prj_filename)
#                RepUtils.createTempFile(xml_file_data, xml_filename)
#
#                gml_data = shp_to_gml(filename=l_filename_tmp, in_schema=str(gml_file_obj.xml_schema_location),
#                                    temp_name=gml_file_obj.id[:gml_file_obj.id.rfind('.')])
#
#                #upload GML data on the GML file
#                gml_file_obj.manage_file_upload(gml_data, 'text/xml')
#
#                #upload/add ESRI files on envelope
#                gml_filename = gml_file_obj.id[:-4]
#                gml_file_title = gml_file_obj.title
#                for k in ESRI_EXTENSIONS:
#                    esri_file_id = '%s.%s' % (gml_filename, k)
#                    esri_file_data = eval('%s_file_data' % k)
#                    if self.testExistingFile(esri_file_id):
#                        esri_file_obj = getattr(self, esri_file_id, None)
#                        esri_file_obj.manage_file_upload(esri_file_data, '')
#                    else:
#                        self.manage_addDocument(esri_file_id, '%s file of the %s' % (k.upper(), gml_file_title),
#                                        esri_file_data, '','')
#
#                #delete temp files
#                for k in ESRI_EXTENSIONS:
#                    RepUtils.deleteTempFile('%s.%s' % (l_filename, k))
#
#                l_msg = 'done'
#            except Exception, e:
#                l_msg = 'Error during conversion! %s' % str(e)
#        else:
#            l_msg = 'Missing mandatory files:'
#            for k in SHORT_ESRI_EXTENSIONS:
#                if not len(eval('%s_filename' % k)):
#                    l_msg = '%s %s file,' % (l_msg, k.upper())
#            l_msg = '%s.' % l_msg[:-1]
#        return l_msg
#
#    security.declareProtected('Change Envelopes', 'convert_ESRI2XML')
#    def convert_ESRI2XML(self, file_id, shx_file, shp_file, dbf_file, meta_file, prj_file, REQUEST=None):
#        """ """
#        msg = self.convert_esri_to_xml(file_id, shx_file, shp_file, dbf_file, meta_file, prj_file)
#        if msg == 'done':
#            REQUEST.RESPONSE.redirect(self.absolute_url())
#        else:
#            REQUEST.RESPONSE.redirect('%s/uploadESRI?file=%s&msg=%s' % (self.absolute_url(), file_id, msg))
#
#    security.declareProtected('Change Envelopes', 'uploadESRI')
#    uploadESRI = DTMLFile('dtml/envelopeUploadESRI',globals())

    ##################################################
    #   Rivers
    ##################################################

    security.declareProtected('Change Envelopes', 'upload_rivers_excel_file')
    upload_rivers_excel_file = DTMLFile('dataflows/Rivers/documentAddExcelData',globals())

    security.declareProtected('Change Envelopes', 'upload_xml_file')
    upload_xml_file = DTMLFile('dataflows/Rivers/documentAddXMLData',globals())

    security.declareProtected('Change Envelopes', 'upload_data_file')
    upload_data_file = DTMLFile('dataflows/Rivers/documentAddOtherData',globals())

    ##################################################
    #   Groundwater
    ##################################################

    security.declareProtected('Change Envelopes', 'fetchXMLFileGW')
    def fetchXMLFileGW(self, REQUEST=None):
        """ Grabs one XML file from a fixed location and uploads it as Report Document
        """
        # Construct the name of the file based on patters and the country code
        # Countries are kept as URI like 'http://cdr.eionet.eu.int/rdf/countries#LY'
        l_file_id_gw = self.getCountryCode().upper() + '_bodies.xml'
        l_file_id_sw = self.getCountryCode().upper() + '_saltwaterintrusion.xml'
        l_file_gw, l_content_type = RepUtils.utGrabFromUrl(WEBQ_XML_REPOSITORY + l_file_id_gw)
        l_file_sw, l_content_type = RepUtils.utGrabFromUrl(WEBQ_XML_REPOSITORY + l_file_id_sw)
        if l_file_gw is None and l_file_sw is None and REQUEST is not None:
            return self.messageDialog(
                                message='There is no initial data available. Use the webforms to create and edit the delivery files.',
                                action=self.absolute_url())
        elif l_file_gw is None and l_file_sw is None:
            return 0

        if hasattr(self, l_file_id_gw):
            self.manage_delObjects(l_file_id_gw)
        if hasattr(self, l_file_id_sw):
            self.manage_delObjects(l_file_id_sw)
        if l_file_gw is not None:
            self.manage_addDocument(id=l_file_id_gw, file=l_file_gw, content_type=l_content_type)
        if l_file_sw is not None:
            self.manage_addDocument(id=l_file_id_sw, file=l_file_sw, content_type=l_content_type)

        if REQUEST is not None:
            return self.messageDialog(
                            message='File(s) successfully fetched!',
                            action=self.absolute_url())
        return 1

    security.declareProtected('Change Envelopes', 'uploadGISfiles')
    def uploadGISfiles(self, file_shp=None, file_shx=None, file_prj=None, file_dbf=None, file_metainfo=None, REQUEST=None):
        """ """
        if file_shp.filename.find('.shp') != -1 and \
                file_shx.filename.find('.shx') != -1 and \
                file_prj.filename.find('.prj') != -1 and \
                file_dbf.filename.find('.dbf') != -1 and \
                file_metainfo.filename.find('.xml') != -1:
            self.manage_addDocument(file=file_shp)
            self.manage_addDocument(file=file_shx)
            self.manage_addDocument(file=file_prj)
            self.manage_addDocument(file=file_dbf)
            self.manage_addDocument(file=file_metainfo)
            if REQUEST is not None:
                return self.messageDialog(
                                message="Files successfully uploaded!",
                                action='.')
            else:
                return 1
        elif file_shp.filename and file_shx.filename and file_prj.filename and \
                file_dbf.filename and file_metainfo.filename:
            if REQUEST is not None:
                return self.messageDialog(
                                message="Files not uploaded! In order for the GIS delivery to be correct and complete, a 'shp', a 'shx', a 'prj', a 'dbf' and an XML file should be present!",
                                action='.')
        else:
            if REQUEST is not None:
                return self.messageDialog(
                                message="Files not uploaded! All fields are mandatory! You must specify all files indicated in order for the GIS delivery to be correct and complete!",
                                action='.')
        return 0

    security.declareProtected('Change Envelopes', 'uploadGISZIPfiles')
    def uploadGISZIPfiles(self, file_gis_zip=None, REQUEST=None):
        """ """
        if file_gis_zip.filename:
            # According to the zipfile.py ZipFile just needs a file-like object
            try:
                zf = ZZipFile(file_gis_zip)
            except:
                if REQUEST is not None:
                    return self.messageDialog(
                                    message="Files not uploaded! The file you have specified is not a zip file!",
                                    action='.')
                else:
                    return 0

            l_file_list = zf.namelist()
            l_mess = ''
            l_extensions = ESRI_EXTRAEXTENSIONS
            for name in l_file_list:
                # test that the archive is not hierarhical
                if name[-1] == '/' or name[-1] == '\\':
                    l_mess = "Files not uploaded! The zip file you specified is hierarchical. It contains folders.\nPlease upload a non-hierarchical structure of files."
                if len(name) > 4:
                    if name[-4:].lower() in l_extensions:
                        l_extensions.remove(name[-4:].lower())
            if l_mess:
                if REQUEST is not None:
                    return self.messageDialog(
                                    message=l_mess,
                                    action='./index_html')
                else:
                    return 0
            # test if all types of files have been added in the archive
            elif len(l_extensions) > 0:
                if REQUEST is not None:
                    return self.messageDialog(
                                    message="Files not uploaded! Not all the files were present in the archive.\nIn order for the GIS delivery to be correct and complete, a 'shp', a 'shx', a 'prj', a 'dbf' and an XML file should be present!",
                                    action='./index_html')
                else:
                    return 0

            for name in zf.namelist():
                zf.setcurrentfile(name)
                self._add_file_from_zip(zf,name, '')

            if REQUEST is not None:
                return self.messageDialog(
                                message="Files successfully uploaded!",
                                action='.')
            else:
                return 1
        else:
            if REQUEST is not None:
                return self.messageDialog(
                                message="You must specify a zip file containing your GIS delivery!",
                                action='.')
            else:
                return 0

    ##################################################
    #   Air Quality Questionnaire
    ##################################################

    def getShapeFiles(self):
        """ Returns all the shape file names from the envelope """
        self.REQUEST.RESPONSE.setHeader('content-type', 'text/xml; charset=utf-8')
        xml = []
        xml_a = xml.append
        xml_a('<?xml version="1.0" encoding="utf-8"?>')
        xml_a("<files>")
        for doc in self.objectValues('Report Document'):
            if doc.id.endswith('.shp'):
                xml_a('<file id="' + doc.id + '" url="' + doc.absolute_url() + '" />')
        xml_a("</files>")
        return ''.join(xml)

    def createRegionInstances(self, x="3", y="28", form_name="form2.xml", language="en"):
        """ Creates XMLs for each region defined in form2.xml e.g. form3_region1.xml"""
        x, y = int(x), int(y)
        #read region names from form2.xml
        regions = []
        dom = self.getFormContentAsXML(form_name)
        rows = dom.getElementsByTagName("form2-row")
        for row in rows:
            region_name = self.getXMLNodeData(row, "region-name")
            if region_name not in regions: regions.append(region_name)

        #for each of the forms from 3 to 27, generate a region specific xml
        region_template = "<reporting-regions>\n\t<region>%s</region>\n</reporting-regions>"
        aqq_empty_instances = self.restrictedTraverse("/", None)["emptyinstances"]["aqq"]
        for xml_number in range(x, y):
            for region in regions:
                xml_filename = "form%s_%s.xml" % (xml_number, region)
                xml_filecontent = aqq_empty_instances["form%s" % xml_number](language)
                xml_filecontent = xml_filecontent.replace("$$$REGION_DATA$$$", region_template % region)
                self.manage_addDocument(xml_filename, "", xml_filecontent, "text/xml", "")

    def getZonesForRegion(self, form_name=None, aqq_rule=None):
      """
      Return the zones available for the specified region and form
      @param form_name This is the name of the instance being edited by the XForm engine (form25...xml etc.).
      Method will retrieve from inside the name of the region within tag <region> and depending on form_name
      will furthermore filter the data according to the chemical compound being edited.
      @param aqq rule to further filter the zones, see b) from ticket. These are hard-coded conventions, for example 'form8b' refers to form 8, second tab
      If no rule is specified, defaults to True, including the zone within the result
      @see https://svn.eionet.europa.eu/projects/Reportnet/ticket/1759
      """
      self.REQUEST.RESPONSE.setHeader('content-type', 'text/xml; charset=utf-8')
      ret = []
      ret.append('<?xml version="1.0" encoding="utf-8"?>')
      ret.append('<response>')
      try:
        form_name = form_name.split('/')[-1]
        #Open the file from request (located in currently editing envelope) and retrieve from it the zones we need to find zones for
        dom = self.getFormContentAsXML(form_name)
        node_rr = dom.getElementsByTagName('reporting-regions')
        req_region_names = []
        if node_rr:
          node = node_rr[0]
          regions = node.getElementsByTagName('region')
          for region in regions:
            req_region_names.append(region.childNodes[0].data)

        #a) Read the form2.xml and extract all necessary codes
        dom = self.getFormContentAsXML('form2.xml')
        zones = dom.getElementsByTagName('form2-row')
        for zone in zones:
          region_name = self.getXMLNodeData(zone, 'region-name')
          if region_name in req_region_names:
            #b) Apply the second type of rule, filter zone by its checked compounds (in form 2)
            if self.matchAQQCompoundRules(zone, aqq_rule):
              ret.append('<zone>')
              ret.append('<name>%s</name>' % self.getXMLNodeData(zone, 'full-zone-name'))
              ret.append('<code>%s</code>' % self.getXMLNodeData(zone, 'zone-code'))
              ret.append('</zone>')
      except:
        traceback.print_exc()
        print 'Error parsing files from envelope: form2.xml or %s' % form_name
      ret.append('</response>')
      return ''.join(ret)


    def matchAQQCompoundRules(self, dom_zone, aqq_rule):
      #Match the AQQ rules see b) from  https://svn.eionet.europa.eu/projects/Reportnet/ticket/1759
      #@param dom_zone minidom xml fragment for a single <form2-row> tag
      #@param form_name Rules are matched based on form_name
      #Rule no. 1 - In form 19 return only the zones having 'O' checked in form 2
      if aqq_rule and aqq_rule.startswith('form19'):
        data = self.getXMLNodeData(dom_zone, 'o')

        return data == 'true'

      #TODO: Other rules
      return True

    def getXMLNodeData(self, domEl, nodeName):
    #Retrieve the data for a node/subnode etc. Works for single nodes only.
        ret = None
        if domEl:
          ret = domEl.getElementsByTagName(nodeName)[0].childNodes[0].data
        return ret

    def getFormContentAsXML(self, form_name):
    #Load an XML from 'report document'/disk into minidom for parsing
      ret = None
      for doc in self.objectValues('Report Document'):
            if doc.id == form_name:
                f = doc.data_file.open()
                ret = f.read()
                f.close()
                break
      return parseString(ret)

# Initialize the class in order the security assertions be taken into account
InitializeClass(EnvelopeCustomDataflows)
