import-email

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from __future__ import print_function

import sys
import os
sys.path.append(os.path.join(sys.path[0],'lib'))

import argparse
import dateutil.parser
import email
import email.header
from email.utils import parseaddr, parsedate
import json
import mailbox
import re
import time

try:
    from urllib import quote_plus
except ImportError:
    from urllib.parse import quote_plus

import n5core.endpoint
from n5core.endpoint import HTTPError
from n5core.pick import pickUnlessOne

# stdnum version 1.10 missing support, it is present in 1.11
from stdnum.no import orgnr

try:
    from stdnum.no.fodselsnummer import is_valid as is_valid_fodselsnummer
except ImportError:
    # Add workaround
    class fnrcheck(object):
        @staticmethod
        def calc_check_digit1(number):
            """Calculate the first check digit for the number."""
            weights = (3, 7, 6, 1, 8, 9, 4, 5, 2)
            return str((11 - sum(w * int(n) for w, n in zip(weights, number))) % 11)

        @staticmethod
        def calc_check_digit2(number):
            """Calculate the second check digit for the number."""
            weights = (5, 4, 3, 2, 7, 6, 5, 4, 3, 2)
            return str((11 - sum(w * int(n) for w, n in zip(weights, number))) % 11)

        @staticmethod
        def is_valid(number):
            """Check if the number is a valid birth number."""
            number = number.strip().replace(' ', '')
            if len(number) != 11:
                return False
            if not number.isdigit():
                return False
            if number[-2] != fnrcheck.calc_check_digit1(number):
                return False
            if number[-1] != fnrcheck.calc_check_digit2(number):
                return False
            return True
    def is_valid_fodselsnummer(number):
        return fnrcheck.is_valid(number)

def decodeheader(s):
    if s is None:
        return s
    retval = None
    for ret, encoding in email.header.decode_header(s):
        if bytes == type(ret):
            if encoding is None:
                ret = ret.decode('ASCII')
            else:
                try:
                    ret = ret.decode(encoding)
                except UnicodeDecodeError:
                    pass # Ignore bogus headers, store them raw
        if retval is None:
            retval = ret
        else:
            retval += ret
    return retval


class MboxImporter(n5core.endpoint.Endpoint):
    def __init__(self):
        baseurl = 'http://localhost:8092/noark5v5/'
        self.mbox = 'input.mbox'
        parser = argparse.ArgumentParser()
        parser.add_argument("--baseurl", help="(default is %s)" % baseurl)
        parser.add_argument("--mda", help="behave as mail delivery agent, read single email from stdin",
                            action="store_true")
        parser.add_argument("--mbox", help="(default is %s)" % self.mbox)
        parser.add_argument("--storageurl", help="specify under which URL (arkivdel/mappe) to store email files in")
        parser.add_argument("--maxinserts", type=int, default=1,
                            help="stop after number of inserts (default is 1)")
        parser.add_argument("--saksmappe",
                            help="Store as saksmappe/journalpost instead of registrering/mappe",
                            action="store_true")
        parser.add_argument("--verbose", help="enable debug output",
                            action="store_true")
        parser.add_argument("--vsm", help="enable VMS fields",
                            action="store_true")
        args = parser.parse_args()
        self.args = args
        self.usesaksmappe = args.saksmappe
        self.storageurl = args.storageurl
        self.maxinserts = args.maxinserts
        if args.baseurl:
            baseurl = args.baseurl
        if args.mda and args.mbox:
            raise Exception("invalid combination of arguments, can not use --mda and --mbox together")
        if args.mbox:
            self.mbox = args.mbox
        self.mda = args.mda
        self.docobjurl = None
        self.parentmap = {}
        self.formatcache = {}
        n5core.endpoint.Endpoint.__init__(self, baseurl)
        self.verbose = args.verbose

    def mime2format(self, mimestr):
        # Observed incorrectly spelled MIME types
        mimealias = {
            'image/jpg': 'image/jpeg',
            'plain/text': 'text/plain',
            'application/x-gzip': 'application/gzip',
        }
        # IANA register at
        # https://www.iana.org/assignments/media-types/media-types.xhtml
        mimetypemap = {
            'application/pdf':           'fmt/95',  # Registered MIME type
            'image/jpeg':                'fmt/41', # Registered MIME type
            'image/gif':                 'fmt/4', # Registered MIME type
            'application/xml':           'fmt/101',  # Registered MIME type
            'text/xml':                  'fmt/101',  # Registered MIME type

            # These should have official codes, but do not
            'application/vnd.oasis.opendocument.presentation' : "fmt/138",
            'application/x-sql':         'fmt/206',  # Not registered MIME type
            'audio/x-wav':               'fmt/703',     # Not registered MIME type
            'image/png':                 'fmt/13',     # Registered MIME type
            'message/rfc822':            'fmt/278',  # Registered MIME type
            'text/csv':                  'x-fmt/18',     # Registered MIME type
            'text/x-diff':               'x-fmt/111',# Not registered MIME type
            'text/x-patch':              'x-fmt/111',# Not registered MIME type

            # No use splitting these formats out as separate upload, the
            # original email content is just fine and we are not going to
            # convert them to archive format anyway.
            'application/pgp-signature':   'ignore',  # Registered MIME type
            'application/pkcs7-signature': 'ignore',  # Registered MIME type
            'application/pgp-keys':        'ignore',  # Registered MIME type
            'multipart/mixed':             'ignore',  # Registered MIME type
            'multipart/signed':            'ignore',  # Registered MIME type
            'multipart/alternative':       'ignore',  # Registered MIME type
            'multipart/related':           'ignore',  # Registered MIME type

            # Do not save as separate uploads if disposition is inline,
            # and otherwise let server guess the format.
            'text/plain':                  None,      # Registered MIME type
            'text/html':                   None,      # Registered MIME type

            # Not quite sure what to do with these, let server guess format.
            'application/x-xz':          None,      # Not registered MIME type
            'application/octet-stream':  None,      # Registered MIME type
            'application/gzip':          None,      # Not registered MIME type
        }
        mimetype = mimestr.split(';')[0]
        if mimetype in mimealias:
            mimetype = mimealias[mimetype]
        if mimetype in mimetypemap:
            format = mimetypemap[mimetype]
            if not self.isKnownFormat(format):
                print("format %s for %s not known by API, asking API to pick format code" % (format, mimetype))
                format = None
            return format
        return None

    def isKnownFormat(self, formatcode):
        if self.formatcache:
            return (formatcode in self.formatcache)
        relkey = self.relbaseurl + 'metadata/format/'

        url = self.findRelation(relkey)
        if url:
            found = False
            (c, r) = self.json_get(url)
            j = json.loads(c)
            if 'results' not in j:
                raise ValueError("unexpected metadata JSON format")
            formatcache = []
            for v in j['results']:
                if formatcode == v['kode']:
                    found = True
                formatcache.append(v['kode'])
            self.formatcache = tuple(formatcache)
            return found
        else:
            raise Exception("unable to look up list of valid formats")

    def createEntity(self, name, rel, rels, data):
        if rel not in rels:
            if self.verbose:
                print(rels)
            raise Exception("unable to find %s in provided relations from %s" % (rel, rels['_links']['self']))
        url = rels[rel]['href']
        try:
            if self.verbose:
                print("GET %s" % url)
            (gc, gres) = self.json_get(url)
            default = json.loads(gc)
            for k in default.keys():
                if not k == '_links' and k not in data:
                    data[k] = default[k]
        except HTTPError as e:
            pass
        try:
            if self.verbose:
                print("POST: %s" % data)
            (c, res) = self.json_post(url, data)
        except HTTPError as e:
            if self.verbose:
                print("POST %s failed: %s (%s)" % (url, str(e), e.read()))
            raise
        info = json.loads(c)
        if self.verbose:
            # Validate the stuff we send came back after storing
            for f in data.keys():
                if data[f] is not None:
                    if  f not in info:
                        print("error: field %s=%s disappeared from object" %  (f, data[f]))
                    elif data[f] != info[f]:
                        print("error: field %s=%s do not match object value %s" %  (f, data[f], info[f]))
        return info

    def getDocObjUrl(self):
        """ Cache href to the list of dokumentobject """
        if not self.docobjurl:
            self.docobjurl = self.findRelation('%sarkivstruktur/dokumentobjekt/' % self.relbaseurl)
        return self.docobjurl

    def findSeries(self):
        """
Figure out which URL to use when creating one file per mail thread.

Store the info about the entity in self.seriesinfo
"""
        if self.storageurl:
            (c, res) = self.json_get(self.storageurl)
            self.seriesinfo = json.loads(c)
            return self.seriesinfo
        fondshref = self.findRelation('%sarkivstruktur/arkiv/' % self.relbaseurl)
        (c, res) = self.json_get(fondshref)
        fondsinfo = json.loads(c)
        info = pickUnlessOne(fondsinfo, 'arkiv')
        if info is None:
            return None

        seriesrel = '%sarkivstruktur/arkivdel/' % self.relbaseurl
        if seriesrel not in info['_links'] \
           or 'href' not in info['_links'][seriesrel]:
            if self.verbose:
                print(info)
            raise Exception("missing from created dokumentobjekt: %s" % seriesrel)
        serieshref = info['_links'][seriesrel]['href']
        if 'templated' in info['_links'][seriesrel] and info['_links'][seriesrel]['templated']:
            serieshref = serieshref.split('{')[0]

        (c, res) = self.json_get(serieshref)
        seriesinfo = json.loads(c)
        #print()
        self.seriesinfo = pickUnlessOne(seriesinfo, 'arkivdel')
        if self.seriesinfo and self.verbose:
            print("serieurl: %s" % self.seriesinfo['_links']['self']['href'])
        return self.seriesinfo

    def parseInReplyTo(self, entry):
        """Split on first semicolon or space and return first part of entry."""
        entry.split(';', 1)
        return re.split(';| ', entry)[0]

    def alreadySaved(self, message):
        """
Look up email Message-Id in dokumentobjekt.filename, and return true
if a match is found.
        """
        self.getDocObjUrl()
        messageid = message['message-id'].strip()
        print("Looking for msgid: %s" % messageid)

        searchurl = self.docobjurl + '?$filter=' +\
            quote_plus("filnavn eq '%s'" % messageid)
        #print(searchurl)

        (s, res) = self.json_get(searchurl)
        docobjsinfo = json.loads(s)
        if 'results' in docobjsinfo:
            if self.verbose:
                print("found message-id %s" % messageid)
            return docobjsinfo['results'][0]
        return None

    def docobj2files(self, docobjinfo):
        """
Given a document object instance, look up the files containing the
document object instance.  This is the structure:

  dokumentobjekt->dokumentbeskrivelse+>registrering->mappe
                                     +>registrering->mappe
                                     +>registrering->mappe

One document might be associated with several records, belonging to
several files.  This function return a list of all the files associated
with a given document object entry.
        """
        if self.verbose:
            print()
            print("Locating mappe for docobj: %s" % docobjinfo)
        docdescrel = '%sarkivstruktur/dokumentbeskrivelse/' % self.relbaseurl
        recordrel = '%sarkivstruktur/registrering/' % self.relbaseurl
        filerel = '%sarkivstruktur/mappe/' % self.relbaseurl

        docdeschref = docobjinfo['_links'][docdescrel]['href']
        files = {}
        if docdeschref:
            print("Found dokumentbeskrivelse %s" % docdeschref)
            (s, res) = self.json_get(docdeschref)
            docdescinfo = json.loads(s)
            recordhref = docdescinfo['_links'][recordrel]['href']
            # Note: one 'dokumentbeskrivelse' can be connected to several 'registrering'
            if recordhref:
                print("Found record list %s" % recordhref)
                (s, res) = self.json_get(recordhref)
                records = json.loads(s)
                if self.verbose:
                    print(records)
                for recordinfo in records['results']:
                    if filerel in recordinfo['_links']:
                        filehref = recordinfo['_links'][filerel]['href']
                        print("Found mappe %s" % filehref)
                        (s, res) = self.json_get(filehref)
                        fileinfo = json.loads(s)
                        fileid = fileinfo['systemID']
                        canonicalfilehref = fileinfo['_links']['self']['href']
                        files[fileid] = canonicalfilehref
        return files

    def findThreadFile(self, message):
        """
Look up if any of the previous messages in a thread (using in-reply-to
and references) are already stored, and if so, return the file they
are in.
"""
        self.getDocObjUrl()
        thread = []
        if 'in-reply-to' in message:
            thread.append(self.parseInReplyTo(message['in-reply-to']))
        if 'references' in message:
            for msgid in message['references'].replace("\n", " ").split(" "):
                if msgid != "":
                    thread.append(msgid.strip())
        if 0 == len(thread):
            return None
        #print(thread)

        fileids = {}
        
        matches = []
        for messageid in thread:
            searchurl = self.docobjurl + '?$filter=' +\
                quote_plus("filnavn eq '%s'" % messageid)
            #print(searchurl)
        
            (s, res) = self.json_get(searchurl)
            docobjsinfo = json.loads(s)
            if self.verbose:
                print()
#                print(docobjsinfo)
                print("checking search result looking for message %s" % messageid)
            if 'results' in docobjsinfo:
                for d in docobjsinfo['results']:
                    if 'filnavn' in d and d['filnavn'] == messageid:
                        matches.append(d['systemID'])
                        if self.verbose:
                            print("info: found message-id %s, locating file" % messageid)
                        files = self.docobj2files(d)
                        if files:
                            if self.verbose:
                                print("info: found file(s) for %s" % messageid)
                            for (fileid, filehref) in files.items():
                                fileids[fileid] = filehref
                        else:
                            print("error: failed to find file for %s" % messageid)

        # FIXME Look up mappe/saksmappe containing docobj entries
        for docobjid in matches:
            if docobjid in self.parentmap:
                (docdescid, childtype, docdeschref) = self.parentmap[docobjid]
                if docdescid in self.parentmap:
                    (recordid, childtype, recordhref) = self.parentmap[docdescid]
                    if recordid in self.parentmap:
                        (fileid, childtype, filehref) = self.parentmap[recordid]
                        fileids[fileid] = filehref
        print("MappeID: ", fileids.keys())
        if 0 < len(fileids.keys()):
            # FIXME what if there are several files?
            filehref = fileids[list(fileids.keys())[0]]
            (fc, fres) = self.json_get(filehref)
            fileinfo = json.loads(fc)
            #print(fileinfo)
            return fileinfo
        return None

    def createNewFile(self, message):
        filedata = {
            'tittel' : decodeheader(message['subject']),
             # FIXME Workaround for bogus ny-* values
            'offentligTittel' : None,
            'beskrivelse' : None,
        }
        if self.usesaksmappe:
            casefilerel = '%ssakarkiv/ny-saksmappe/' % self.relbaseurl
            fileinfo = self.createEntity('saksmappe', casefilerel,
                                         self.seriesinfo['_links'],
                                         filedata)
        else:
            filerel = '%sarkivstruktur/ny-mappe/' % self.relbaseurl
            fileinfo = self.createEntity('mappe', filerel,
                                         self.seriesinfo['_links'],
                                         filedata)
        return fileinfo
    
    def recordMapping(self, parentinfo, childinfo, type):
        childid = childinfo['systemID']
        parentid = parentinfo['systemID']
        parenthref = parentinfo['_links']['self']['href']
        self.parentmap[childid] = (parentid, type, parenthref)


    def createDocumentObject(self, docdescinfo, docobjdata, content, created):
        docobjrel = '%sarkivstruktur/ny-dokumentobjekt/' % self.relbaseurl
        docobjinfo = self.createEntity('dokumentobjekt', docobjrel,
                                       docdescinfo['_links'],
                                       docobjdata)
        created.append(docobjinfo['_links']['self']['href'])
        self.recordMapping(docdescinfo, docobjinfo, 'dokumentobjekt')

        uploadrel = '%sarkivstruktur/fil/' % self.relbaseurl
        if uploadrel not in docobjinfo['_links'] \
           or 'href' not in docobjinfo['_links'][uploadrel]:
            if self.verbose:
                print(docobjinfo)
            raise Exception("missing from created dokumentobjekt: %s" % uploadrel)

        newfilehref = docobjinfo['_links'][uploadrel]['href']

        # Make sure all parts going into the POST are type str by
        # converting newfilehref to str.  This avoid a
        # UnicodeDecodeError when the content is not UTF-8.
        try:
            (c, res) = self.post(str(newfilehref), content,
                                 docobjdata['mimeType'],
                                 length=len(content))
        except Exception as e:
            print("error: upload threw exception, removing object: %s" % e)
            for o in created[::-1]:
                self.delete(o)
            return False
        if 201 != res.code:
            print("error: upload failed (status == %d), removing objects"
                  % res.code)
            for e in created[::-1]:
                print("info: deleting %s" % e)
                self.delete(e)
            return False
        return True


    def rfc822date2xsdate(self, s):
        """Convert a RFC 822 date string to a XML xs:date string,
not loosing the timezone information.

"""
        d = dateutil.parser.parse(s)
        if d.tzinfo:
            return d.isoformat()
        else:
            return d.isoformat() + 'Z'


    def locateIdentifiers(self, content, found = None):
        """Locate Norwegian person numbers, organisation numbers and other
identifiers in email content.  Person numbers are 11 digits, the first
6 are the birth date, the last 5 are sequence numbers, gender
indicator and control digit.  Organisation numbers are 9 digits, the
last digit is a control digit.

        """

        if found is None:
            found = []
        # Match only 9 and 11 digit numbers, possibly with some
        # spacing allowed.  In other words:
        # 30093426645
        # 261249 14484
        # 998787173
        # 813 905 582
        p = r"""(?x)
                (?<!\d)
                \b(\d{9}|\d{6}\s\d{5}|\d{11}|\d{3}\s\d{3}\s\d{3})\b
                (?!\d)
"""
        if bytes == type(content):
            p = p.encode('UTF-8')
        matches = re.findall(p, content)
        for m in matches:
            if bytes == type(m):
                m = m.decode('UTF-8')
            m = m.replace(" ", "")
            digits = len(m)
            if 9 == digits and orgnr.is_valid(m):
                data = {
                    'enhetsidentifikator': { 'organisasjonsnummer': m }
                }
                if data not in found: found.append(data)
            elif 11 == digits and is_valid_fodselsnummer(m):
                data = {
                    'foedselsnummer': { 'foedselsnummer': m }
                }
                if data not in found: found.append(data)
        return found


    def addMessageToFile(self, fileinfo, message):
        recordrel = '%sarkivstruktur/ny-registrering/' % self.relbaseurl
        recordentryrel = '%ssakarkiv/ny-journalpost/' % self.relbaseurl
        docdescrel = '%sarkivstruktur/ny-dokumentbeskrivelse/' % self.relbaseurl
        createkeywordrel = self.nikitarelbaseurl + 'ny-noekkelord/'

        created = []
        content = message.as_bytes()
        body = message.as_string().split("\n\n", 1)[1]
        subject = message['subject']
        identifiers = self.locateIdentifiers(body)
        identifiers = self.locateIdentifiers(subject, found=identifiers)
        messagelen = len(content)
        mimetype = "message/rfc822"
        if (message['mime-version'] and message['content-type']):
            format = "fmt/950"
        else:
            format = "fmt/278"

        threadstarterid = None
        if 'references' in message:
            threadstarterid = message['references'].strip().split()[0]
        elif 'in-reply-to' in message:
            threadstarterid = self.parseInReplyTo(message['in-reply-to'])
        else:
            threadstarterid = message['message-id'].strip()

        if self.usesaksmappe:
            recorddata = {
                'tittel' : decodeheader(subject),
                # FIXME figure out way to detect more sensible type
                # for incoming and internal email.
                'journalposttype'      : {'kode': 'I',
                                          'kodenavn': 'Inngående dokument'},
                # Workaround for bogus ny-* values
                'beskrivelse' : None,
            }
            if 'date' in message:
                fromdate = self.rfc822date2xsdate(message['date'])
                recorddata['dokumentetsDato'] = fromdate
                recorddata['sendtDato'] = fromdate # what if date is faked?

            recordinfo = self.createEntity('journalpost', recordentryrel,
                                           fileinfo['_links'],
                                           recorddata)
            created.append(recordinfo['_links']['self']['href'])
            self.recordMapping(fileinfo, recordinfo, 'journalpost')
            # FIXME consider journalpost and korrespondansepart (from
            # to/from/cc)
            corrpartrel = '%sarkivstruktur/ny-korrespondansepartperson/' % self.relbaseurl
            parts = {
                'from' : 'EA',
                'to'   : 'EM',
                'cc'   : 'EK',
            }
            # FIXME should sort into KorrespondansepartEnhet,
            # KorrespondansepartPerson and KorrespondansepartIntern.
            # How can we do this?
            for part in parts:
                if part in message:
                    partstr = decodeheader(message[part])
                    for p in partstr.split(","):
                        identifiers = \
                            self.locateIdentifiers(p, found=identifiers)
                        navn, epost = parseaddr("From: %s" % p)
                        if not navn:
                            navn = epost.split('@')[0]
                        corrdata = {
                            'korrespondanseparttype' : {
                                'kode' : parts[part],
                            },
                            'navn' : navn,
                            'kontaktinformasjon' : { 'epostadresse' : epost},
#                            'kontaktperson' : None,
#                            'forretningsadresse' : {},
#                            'postadresse' : {},
                        }
                        corrinfo = \
                            self.createEntity('correspondance part',
                                              corrpartrel,
                                              recordinfo['_links'],
                                              corrdata)

        else:
            recorddata = {
                'tittel' : decodeheader(message['subject']),
                'beskrivelse' : None,
            }
            recordinfo = self.createEntity('registrering', recordrel,
                                           fileinfo['_links'],
                                           recorddata)
            created.append(recordinfo['_links']['self']['href'])
            self.recordMapping(fileinfo, recordinfo, 'registrering')

        # Flag new email records using keyword 'new' to make it
        # possible to locate them for post-processing after import.
        keyworddata = {
            'noekkelord': 'new',
        }
        keywordinfo = \
            self.createEntity('keyword', createkeywordrel,
                              recordinfo['_links'], keyworddata)
        
        docdescdata = {
            'tittel' : decodeheader(message['subject']),
            'oppbevaringssted' : threadstarterid ,
        }
        if self.args.vsm:
            docdescdata['virksomhetsspesifikkeMetadata'] = {
                'vnd-email-v1:thread': threadstarterid,
                'vnd-email-v1:message-id': message['message-id'].strip(),
            }

        docdescinfo = self.createEntity('dokumentbeskrivelse', docdescrel,
                                        recordinfo['_links'],
                                        docdescdata)
        created.append(docdescinfo['_links']['self']['href'])
        self.recordMapping(recordinfo, docdescinfo, 'dokumentbeskrivelse')
        
        docobjdata = {
            "versjonsnummer"  : 0,
            'variantformat'   : {
                'kode': 'P',
                'kodenavn': 'Produksjonsformat'
            },
            "filstoerrelse"   : messagelen,
            "filnavn"         : message['message-id'].strip(),       
            "mimeType"        : mimetype,
        }
        if format:
            docobjdata["format"] = { 'kode': format, }
        if not self.createDocumentObject(docdescinfo, docobjdata,
                                         content, created):
            print("error: failed to create document object")
            return False
        if message.is_multipart():
            for part in message.walk():
                mimetype = part.get_content_type()
                format = self.mime2format(mimetype)
                try: # Only available with Python 3
                    disposition = part.get_content_disposition()
                except AttributeError as e:
                    # Assume attachment on python 2
                    disposition = 'attachment'
                if self.verbose:
                    print("info: mapping %s to %s" % (mimetype, format))

                # Do not store ignored attachments as separate
                # documents.  Everything else we upload and let the
                # kernel figure out what to do with them. :)
                if format is not None and 'ignore' == format:
                    continue

                # If it is unknown what to do with the format and the
                # disposition is inline, do not create separate
                # document as we assume it is enough to keep it as
                # part of the original RFC822 message.
                if format is None and 'inline' == disposition:
                    continue

                content = part.get_payload(decode=True)
                if content is None or content == "":
                    continue
                self.locateIdentifiers(content, identifiers)

                docdescdata = {
                    'tittel' : decodeheader(message['subject']),
                    'oppbevaringssted' : threadstarterid ,
                }
                docdescinfo = self.createEntity('dokumentbeskrivelse', docdescrel,
                                            recordinfo['_links'],
                                                docdescdata)
                created.append(docdescinfo['_links']['self']['href'])
                self.recordMapping(recordinfo, docdescinfo, 'dokumentbeskrivelse')

                contentlen = len(content)
                docobjdata = {
                    "versjonsnummer"  : 0,
                    'variantformat'   : {
                        'kode': 'P',
                        'kodenavn': 'Produksjonsformat'
                    },
                    "filstoerrelse"   : contentlen,
                    "filnavn"         : part.get_filename(),
                    "mimeType"        : mimetype,
                }
                if format:
                    docobjdata["format"] = { "kode": format, }

                if not self.createDocumentObject(docdescinfo, docobjdata,
                                                 content, created):
                    print("error: failed to create document object for attachment %s"
                          % part.get_filename())

        print("Identifiers:", identifiers)
        for identifier in identifiers:
            for idtype in identifier.keys():
                data = identifier[idtype]
                rel = '%sarkivstruktur/ny-%s/' % (self.relbaseurl, idtype)
                iddata = self.createEntity(idtype, rel,
                                           recordinfo['_links'],
                                           data)
        return True

    def uploadEmail(self, message):
        docobjinfo = self.alreadySaved(message)
        if docobjinfo:
            print("email already part of archive")
            return False
        threadfile = self.findThreadFile(message)
        if not threadfile:
            # create new file
            threadfile = self.createNewFile(message)

        # Add to existing file
        return self.addMessageToFile(threadfile, message)

    def createVSM(self):
        if not self.args.vsm:
            return
        metadatarel = self.relbaseurl + 'metadata/'
        vsmrel = "%smetadata/virksomhetsspesifikkeMetadata/" \
            % self.relbaseurl
        createvsmrel = "%smetadata/ny-virksomhetsspesifikkeMetadata/" \
            % self.relbaseurl
        url = self.findRelation(metadatarel)
        (entry, res) = self.json_get(url)
        metadatalist = json.loads(entry)

        try:
            vsmhref = metadatalist['_links'][vsmrel]['href']
            (entry, res) = self.json_get(vsmhref)
            vsmlist = json.loads(entry)
        except HTTPError as e:
            print("GET %s failed: %s (%s)" % (url, str(e), e.read()))
            raise
        vsmused = (
            ('vnd-email-v1:thread', 'string',
             'Message-ID of first or previous email in email thread.'),
            ('vnd-email-v1:message-id', 'string',
             'Message-ID of archived email.'),
        )
        for vsmkey, vsmtype, vsmdesc in vsmused:
            # Only create if missing
            found = False
            if 0 < vsmlist['count']:
                for v in vsmlist['results']:
                    if vsmkey == v['navn']:
                        if vsmtype == v['type']:
                            found = True
                            break
                        else:
                            raise Exception("Conflicting VSM type for %s (%s, not %s)"
                                            % (vsmkey, v['type'], vsmtype))
            if not found:
                vsmdata = {
                    u'navn' : vsmkey,
                    u'type' : vsmtype,
                    u'beskrivelse' : vsmdesc,
                    #'kilde' : '',
                    #'utdatert' : False,
                }
                try:
                    vsmmetainfo = \
                        self.createEntity('virksomhetsspesifikkeMetadata:%s'
                                          % vsmkey, createvsmrel,
                                          metadatalist['_links'], vsmdata)
                except HTTPError as e:
                    print("error: unable to create required VSM entry, exiting: %s (%s)"
                          % (str(e), e.read()))
                    praise

    def loadMbox(self):
        retval = True
        mbox = mailbox.mbox(self.mbox)
        self.createVSM()
        last = time.time()
        count = 0
        inserted = 0
        for message in sorted(mbox,
                              key=lambda email: parsedate(email.get('Date'))):
            count += 1
            if not self.uploadEmail(message):
                print("error: not loading email %s" % message['message-id'].strip())
                retval = False
            else:
                inserted += 1
            now = time.time()
            print("%.2f s per insert/email (%d inserted, %d seen)" % (
                now - last, inserted, count
            ))
            last = now
            if inserted >= self.maxinserts:
                break
        return retval

    def loadEmail(self):
        """ Load a single email from stdin like /usr/sbin/sendmail does. """
        message = email.message_from_file(sys.stdin)
        self.createVSM()
        if not self.uploadEmail(message):
            print("error: not loading email %s" % message['message-id'].strip())
            return False
        return True

    def load(self):
        if self.mda:
            return self.loadEmail()
        else:
            return self.loadMbox()

def main():
    t = MboxImporter()
    t.login()
    if t.findSeries() is None:
        print("error: unable to find arkivdel to use")
        return 1
    return t.load() != True

if __name__ == '__main__':
    exit(main())