ldpdb-import

#! /usr/bin/python
# 
# This file is part of the Lampadas Documentation System.
# 
# Copyright (c) 2000, 2001, 2002 David Merrill <david@lupercalia.net>.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
# 
# NOTE: This script is intended to do the upgrade from the old LDPDB
# system into Lampadas. So, it has lots of special-purpose code.
# This file should not be included in any actual release of Lampadas.
# 
from Globals import *
from Database import get_database
from BaseClasses import *
import sys
import os
import string

DEBUG = 0

DOCUMENT_LANGUAGES = {68:   'FI',   # Finnish HOWTO
                      87:   'EL',   # Hellenic HOWTO
                      483:  'LV',   # Latvian HOWTO
                      124:  'FR',   # Francophone HOWTO
                      197:  'PT',   # Purtuguese HOWTO
                      151:  'EO',   # Esperanto HOWTO
                      283:  'SH',   # Serbian HOWTO
                      479:  'SK',   # Slovak HOWTO
                      290:  'SL',   # Slovenian HOWTO
                      297:  'ES',   # Spanish HOWTO
                      384:  'TK'}   # Turkish HOWTO

class Name:

    def __init__(self, fullname):
        name = fullname
        name = name.replace('Dr\.', '')
        name = name.replace(', II', '')
        name = name.replace(', III', '')
        name = string.split(name)
        if len(name)==1:
            self.first_name  = name[0]
            self.middle_name = ''
            self.surname     = ''
        elif len(name)==2:
            self.first_name  = name[0]
            self.middle_name = ''
            self.surname     = name[1]
        elif len(name)==3:
            self.first_name  = name[0]
            self.middle_name = name[1]
            self.surname     = name[2]
        elif len(name)==4:
            self.first_name  = name[0]
            mi1              = name[1]
            mi2              = name[2]
            self.middle_name = mi1 + ' ' + mi2
            self.surname     = name[3]
            
            if mi2.upper() in ('DE','DER','DOS','VAN','DA'):
                self.surname = mi2 + self.surname
                self.middle_name = mi1
        else:
            print "ERROR: found a five word name, cannot parse it: " + string.join(name)
            sys.exit()
        
        if self.middle_name.upper() in ('VAN','VAN DER','VAN DEN','DE','DE LA',
                                        'DER','DOS','DA'):
            self.surname = self.middle_name + ' ' + self.surname
            self.middle_name = ''
        
    
class OldMaintainers(LampadasCollection):
   
    def __init__(self):
        self.data = {}
        sql = 'SELECT maintainer_id, maintainer_name, email FROM maintainer'
        cursor = ldp_db.select(sql)
        while (1):
            row = cursor.fetchone()
            if row==None: break
            maintainer = OldMaintainer()
            maintainer.load_row(row)
            self[maintainer.id] = maintainer


class OldMaintainer:

    def load_row(self, row):
        self.id    = row[0]
        self.name  = trim(row[1])
        self.email = trim(row[2])
        name = Name(self.name)
        self.first_name  = name.first_name
        self.middle_name = name.middle_name
        self.surname     = name.surname
        

class OldEditors(LampadasCollection):
    
    def __init__(self):
        self.data = {}
        sql = 'SELECT editor_id, editor_name, email, notes FROM editor'
        cursor = ldp_db.select(sql)
        while (1):
            row = cursor.fetchone()
            if row==None: break
            editor = OldEditor()
            editor.load_row(row)
            self[editor.id] = editor


class OldEditor:

    def load_row(self, row):
        self.id    = row[0]
        self.name  = trim(row[1])
        self.email = trim(row[2])
        self.notes = trim(row[3])

        name = Name(self.name)
        self.first_name  = name.first_name
        self.middle_name = name.middle_name
        self.surname     = name.surname


class OldUsers(LampadasCollection):
    
    def __init__(self):
        self.data = {}
        sql = 'SELECT username, first_name, surname, email, admin, '\
              '       maintainer_id, editor_id '\
              'FROM username'
        cursor = ldp_db.select(sql)
        while (1):
            row = cursor.fetchone()
            if row==None: break
            user = OldUser()
            user.load_row(row)
            self[user.username] = user

class OldUser:

    def load_row(self, row):
        self.username      = string.lower(trim(row[0]))
        self.first_name    = trim(row[1])
        self.surname       = trim(row[2])
        self.email         = trim(row[3])
        self.admin         = tf2bool(row[4])
        self.maintainer_id = row[5]
        self.editor_id     = row[6]


class NewUsers(LampadasCollection):

    def save(self):
        keys = self.keys()
        for key in keys:
            user = self[key]
            user.save()

    def find_maintainer(self, maintainer_id):
        keys = self.keys()
        for key in keys:
            user = self[key]
            if user.maintainer_id==maintainer_id:
                return user
        

    def find_editor(self, editor_id):
        keys = self.keys()
        for key in keys:
            user = self[key]
            if user.editor_id==editor_id:
                return user

class NewUser:

    def __init__(self):
        self.username      = ''
        self.first_name    = ''
        self.middle_name = ''
        self.surname       = ''
        self.email         = ''
        self.admin         = 0
        self.notes         = ''
        self.password      = random_string(12)
        self.maintainer_id = 0
        self.editor_id     = 0

    def username_from_name(self):
        self.username = trim(self.first_name + trim(self.middle_name + self.surname))
        self.username = self.username.lower()
        self.username = self.username.replace('&','')
        self.username = self.username.replace('.','')
        self.username = self.username.replace('\'','')
        self.username = self.username.replace(' ','')
        
    def username_from_email(self):
        self.username = self.email
        
    def save(self):
        sql = 'INSERT INTO username(username, first_name, middle_name, surname, '
        sql += 'email, admin, notes, password) '
        sql += 'VALUES(' + wsq(self.username) + ', ' + wsq(self.first_name) + ', ' + wsq(self.middle_name) + ', ' + wsq(self.surname) + ', '
        sql += wsq(self.email) + ', ' + wsq(bool2tf(self.admin)) + ', ' + wsq(self.notes) + ', ' + wsq(self.password) + ')'
        lampadas_db.runsql(sql)
        lampadas_db.commit()


###################################################################3

def copy_users():
    # Copy users over
    for username in usernames:
        user = users[username]
        newuser = NewUser()
        newuser.username      = user.username
        newuser.first_name    = user.first_name
        newuser.surname       = user.surname
        newuser.email         = user.email
        newuser.admin         = user.admin
        newuser.maintainer_id = user.maintainer_id
        newuser.editor_id     = user.editor_id
        newusers[newuser.username]    = newuser

    # Merge maintainer data
    for maintainer_id in maintainer_ids:
        maintainer = maintainers[maintainer_id]
        newuser = newusers.find_maintainer(maintainer_id)
        if newuser:
            newuser.maintainer_id = maintainer_id
            if newuser.email=='':
                newuser.email = maintainer.email
        else:
            newuser = NewUser()
            newuser.first_name    = maintainer.first_name
            newuser.middle_name   = maintainer.middle_name
            newuser.surname       = maintainer.surname
            newuser.email         = maintainer.email
            newuser.maintainer_id = maintainer.id
            newuser.username_from_name()
            if newusers.has_key(newuser.username):
                print 'NOTICE: maintainer username is taken: ' +newuser.username
                newuser.username_from_email()
                print '           using email changed it to: ' + newuser.username
                if newusers[newuser.username]:
                    print "ERROR: cannot generate unique username for: " + newuser.username + ', ' + newuser.name
                    sys.exit()
            newusers[newuser.username] = newuser
    
    # Merge editor data
    for editor_id in editor_ids:
        editor = editors[editor_id]
        newuser = newusers.find_editor(editor_id)
        if newuser:
            newuser.editor_id = editor_id
            newuser.notes = editor.notes
            if newuser.email=='':
                newuser.email = editor.email
        else:
            newuser = NewUser()
            newuser.first_name  = editor.first_name
            newuser.middle_name = editor.middle_name
            newuser.surname     = editor.surname
            newuser.email       = editor.email
            newuser.notes       = editor.notes
            newuser.editor_id   = editor.id
            newuser.username_from_name()
            if newusers.has_key(newuser.username):
                print 'NOTICE: editor username is taken: ' +newuser.username
                newuser.username_from_email()
                print '       using email changed it to: ' + newuser.username
                if newusers[newuser.username]:
                    print "ERROR: cannot generate unique username for: " + newuser.username + ', ' + newuser.name
                    sys.exit()
            newusers[newuser.username] = newuser

    # Save the new data
    newusers.save()
    
    # Reset david's information -- just for my own convenience.
    sql = 'UPDATE username SET sysadmin=' + wsq('t') + ', password=' + wsq('byteme') + ' WHERE username=' + wsq('david')
    lampadas_db.runsql(sql)
    lampadas_db.commit()

def copy_documents():
    sql = 'SELECT doc_id, title, filename, class, format, dtd, dtd_version, version, last_update, url, isbn, pub_status, review_status, tickle_date, pub_date, ref_url, tech_review_status, maintained, license, abstract, rating FROM document'
    cursor = ldp_db.select(sql)
    while (1):
        row = cursor.fetchone()
        if row==None: break
        # adjust doc_id to lie above any existing documents
        doc_id              = row[0] + doc_id_offset
        title               = trim(row[1])
        filename            = trim(row[2])
        type                = trim(row[3])
        format              = trim(row[4])
        dtd                 = trim(row[5])
        dtd_version         = trim(row[6])
        version             = trim(row[7])
        last_update         = date2str(row[8])
        url                 = trim(row[9])
        isbn                = trim(row[10])
        orig_pub_status     = trim(row[11])
        review_status       = trim(row[12])
        tickle_date         = date2str(row[13])
        pub_date            = date2str(row[14])
        ref_url             = trim(row[15])
        tech_review_status  = trim(row[16])
        maintained          = tf2bool(row[17])
        license             = trim(row[18])
        abstract            = trim(row[19])
        rating              = safeint(row[20])

        if DOCUMENT_LANGUAGES.has_key(doc_id):
            lang = DOCUMENT_LANGUAGES[doc_id]
        else:
            lang = 'EN'

        type = type.lower()
        if type=='guide':
            type = 'userguide'
        if type=='mini':
            type = 'howto'
        if type=='backgrounder':
            type = 'intro'
        if type=='quick':
            type = 'quickref'

        format = format.lower()
        if format=='wiki':
            format = 'wikitext'

        if dtd=='N/A':
            dtd = 'none'
        if dtd=='HTML':
            dtd = 'html'
        if dtd=='LinuxDoc':
            dtd = 'linuxdoc'
        if dtd=='DocBook':
            dtd = 'docbook'
            
        # Lampadas doesn't understand LDP-specific licensing
        # to this degree.
        license = license.lower()
        if license=='boilerplate':
            license = 'free'
        if license=='howtol' or license=='ldpcl' or license=='oldldpl':
            license = 'ldpl'
       
        pub_status = orig_pub_status

        # unknown status should just be NULL
        if pub_status=='?':
            pub_status = ''
        
        # offsite records do not get copies.
        # Record doc_id in ignore_docs so child data is ignored too.
        if pub_status=='O':
            ignore_docs.append(doc_id)
            continue
        
        # replaced pub_status is gone -- it wasn't really a pub
        # status. The status for these is deleted, with a separate
        # field to indicate what document replaced it.
        if pub_status=='R':
            pub_status = 'D'

        if pub_status=='A':
            pub_status='N'

        # Calculate a sk_seriesid:
        sk_seriesid = new_sk_seriesid()

        # Date fields are different:
        #
        #   pub_date -> first_pub_date
        #   last_update -> pub_date
        #   last_update dropped, it is now the newest file date.
        #
        sql = 'INSERT INTO document(doc_id, lang, title, type_code, format_code, dtd_code, dtd_version, version, '
        sql += 'isbn, pub_status_code, review_status_code, tickle_date, pub_date, tech_review_status_code, maintained, '
        sql += 'license_code, abstract, rating, sk_seriesid, first_pub_date) '
        sql += 'VALUES(' + str(doc_id) + ', ' + wsq(lang) + ', ' + wsq(title) + ', ' + wsq(type) + ', ' + wsq(format) + ', ' + wsq(dtd) + ', ' + wsq(dtd_version) + ', ' + wsq(version) + ', '
        sql += wsq(isbn) + ',' + wsq(pub_status) + ', ' 
        sql += wsq(review_status) + ', ' + wsq(tickle_date) + ', ' 
        sql += wsq(last_update) + ', '
        sql += wsq(tech_review_status) + ', ' + wsq(bool2tf(maintained)) + ', '
        sql += wsq(license) + ', ' + wsq(abstract) + ', ' + str(rating) + ', ' + wsq(sk_seriesid) + ', ' + wsq(pub_date) + ')'
        lampadas_db.runsql(sql)
        if orig_pub_status=='A':
            sql = "INSERT INTO document_collection(doc_id, collection_code) VALUES(" + str(doc_id) + ", 'ldp-archive')"
            lampadas_db.runsql(sql)
            sql = "UPDATE document SET pub_status_code='N' where doc_id=" + str(doc_id)
            lampadas_db.runsql(sql)
        else:
            sql = "INSERT INTO document_collection(doc_id, collection_code) VALUES(" + str(doc_id) + ", 'ldp')"
            lampadas_db.runsql(sql)
        lampadas_db.commit()

    # Set HOWTO-HOWTO as replaced by LDP Author Guide
    sql = 'UPDATE document SET replaced_by_id=121 WHERE doc_id=90'
    lampadas_db.runsql(sql)
    lampadas_db.commit()

    # Set Boca-HOWTO as replaced by Installing Boca Card Mini-HOWTO
    sql = 'UPDATE document SET replaced_by_id=103 WHERE doc_id=20'
    lampadas_db.runsql(sql)
    lampadas_db.commit()

    # Delete HOWTO-Index.
    sql = 'UPDATE document SET pub_status_code=' + wsq('D') + ' WHERE doc_id=122'
    lampadas_db.runsql(sql)
    lampadas_db.commit()
    
    # Add the extra files for documents that have more than one.
    add_sourcefile(3, 'guide/docbook/abs-guide/add-drive.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/alias.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/allprofs.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/alt-bc.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/am-i-root.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/and-list3.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/and-or.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/arglist.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/arith-ops.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/arith-tests.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/array-function.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/assert.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/background-loop.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/badread.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/base.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/bashandperl.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/bashrc')
    add_sourcefile(3, 'guide/docbook/abs-guide/behead.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/bin-grep.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/blank-rename.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/blot-out.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/break-levels.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/bubble.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/c-vars.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/case-cmd.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/col-totaler.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/col-totaler3.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/col.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/collatz.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/colm.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/commentblock.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/connect-stat.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/continue-nlevel.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/copy-cd.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/crypto-quote.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/csubloop.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/cvt.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/data-file')
    add_sourcefile(3, 'guide/docbook/abs-guide/days-between.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/dd-keypress.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/de-rpm.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/du.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/empty-array.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/encryptedpw.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/erase.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/escaped.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex1.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex10.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex11.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex13.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex14.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex15.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex16.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex17.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex18.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex19.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex3.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex30.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex31.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex33.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex33a.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex34.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex35.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex36.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex36a.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex37.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex38.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex39.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex4.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex40.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex41.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex43.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex44.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex45.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex45a.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex46.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex47.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex48.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex49.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex5.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex50.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex51.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex53.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex54.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex55.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex56.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex57.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex58.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex59.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex6.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex60.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex61.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex63.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex64.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex65.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex66.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex67.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex68.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex69.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex7.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex70.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex71.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex71a.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex71b.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex71c.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex73.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex74.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex75.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex76.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex77.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex78.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex79.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex8.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ex9.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/factr.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/fifo.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/file-comparison.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/file-info.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/file-integrity.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/findstring.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/for-loopc.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/for-loopcmd.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ftpget.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/gcd.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/gen0')
    add_sourcefile(3, 'guide/docbook/abs-guide/grp.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/here-function.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/hexconvert.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/hypotenuse.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/idelete.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ifs-empty.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ifs.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/incompat.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ind-ref.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/int-or-string.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/isalpha.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/keypress.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/kill-process.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/length.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/life.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/line-number.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/list-glob.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/logevents.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/lookup.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/lowercase.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/m4.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/mail-format.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/makedict.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/manview.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/match-string.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/max.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/max3.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/missing-keyword.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/monthlypmt.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/multiplication.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/names.data')
    add_sourcefile(3, 'guide/docbook/abs-guide/nested-loop.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/numbers.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/obj-oriented.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/online.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/param-sub.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/patt-matching.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/pb.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/pid-identifier.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/primes.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/pw.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/q-function.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ramdisk.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/random-test.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/random3.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/read-novar.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/read-r.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/read-redir.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/realname.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/reassign-stdout.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/recurse.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/redir1.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/redir3.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/redir3a.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/redir4.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/redir4a.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/redir5.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/ref-params.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/reply.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/resistor-inventory.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/restricted.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/return-test.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/rfe.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/rn.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/rnd.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/rot13.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/rot13_3.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/rot14.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/rpm-check.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/script-detector.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/seconds.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/secret-pw.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/seeding-random.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/self-destruct.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/self-document.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/self-exec.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/self-mailer.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/set-pos.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/stack.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/str-test.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/string.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/strip-comments.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/stupid-script-tricks.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/subshell-pitfalls.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/subshell.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/substring-extraction.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/sum-product.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/symlinks.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/symlinks3.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/t-out.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/timed-input.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/timeout.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/tree.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/twodim.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/unalias.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/unset.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/upperconv.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/userlist.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/var-match.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/vartrace.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/viewdata.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/weirdvars.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/wf.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/wh-loopc.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/what.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/wipedir.sh')
    add_sourcefile(3, 'guide/docbook/abs-guide/words.data')
    add_sourcefile(3, 'guide/docbook/abs-guide/wstrings.sh')

def add_sourcefile(doc_id, filename):
    sql = 'INSERT INTO sourcefile(filename) VALUES (' + wsq(filename) + ');'
    if DEBUG > 0:
        print sql
    lampadas_db.runsql(sql)
    sql = 'INSERT INTO document_file(doc_id, filename) VALUES (' + str(doc_id) + ', ' + wsq(filename) + ');'
    if DEBUG > 0:
        print sql
    lampadas_db.runsql(sql)

def copy_document_users():
    sql = 'SELECT doc_id, maintainer_id, role, active, email FROM document_maintainer'
    cursor = ldp_db.select(sql)
    while (1):
        row = cursor.fetchone()
        if row==None: break
        # adjust doc_id to lie above any existing documents
        doc_id        = row[0] + doc_id_offset
        
        # Ignore child data if parent was ignored.
        if doc_id in ignore_docs:
            continue

        maintainer_id = row[1]
        role          = trim(row[2])
        active        = tf2bool(row[3])
        email         = trim(row[4])

        role = role.lower()
        role = role.replace('-','')

        user = newusers.find_maintainer(maintainer_id)
        username = user.username

        sql = 'INSERT INTO document_user (doc_id, username, role_code, email, active) '
        sql += 'VALUES (' + str(doc_id) + ', ' + wsq(username) + ', ' + wsq(role) + ', ' 
        sql += wsq(email) + ', ' + wsq(bool2tf(active)) + ')'
        if DEBUG > 0:
            print sql
        lampadas_db.runsql(sql)
        lampadas_db.commit()

def copy_document_topics():
    sql = 'SELECT doc_id, topic_num, subtopic_num FROM document_topic'
    cursor = ldp_db.select(sql)
    while (1):
        row = cursor.fetchone()
        if row==None: break
        # adjust doc_id to lie above any existing documents
        doc_id        = row[0] + doc_id_offset

        # Ignore child data if parent was ignored.
        if doc_id in ignore_docs:
            continue
        
        topic_num     = row[1]
        subtopic_num  = row[2]

        # Find the topic. It must exist.
        sql = 'SELECT topic_code'
        sql += ' FROM topic'
        sql += ' WHERE sort_order/10000 =' + str(topic_num)
        sql += ' AND mod(sort_order, 10000)/100 =' + str(subtopic_num)
        cursor2 = lampadas_db.select(sql)
        row2 = cursor2.fetchone()
        if row2==None:
            print 'ERROR: could not locate the topic_code for topic_num: ' + str(topic_num) + ', subtopic_num: ' + str(subtopic_num)
            sys.exit()
        topic_code = trim(row2[0])

        sql = 'INSERT INTO document_topic(doc_id, topic_code) '
        sql += 'VALUES (' + str(doc_id) + ', ' + wsq(topic_code) + ')'
        lampadas_db.runsql(sql)
        lampadas_db.commit()


def copy_document_revs():
    rev_id = lampadas_db.max_id('document_notes', 'note_id')
    sql = 'SELECT doc_id, version, pub_date, initials, notes FROM document_rev'
    cursor = ldp_db.select(sql)
    while(1):   
        row = cursor.fetchone()
        if row==None: break
        # adjust doc_id to lie above any existing documents
        doc_id   = row[0] + doc_id_offset
        
        # Ignore child data if parent was ignored.
        if doc_id in ignore_docs:
            continue

        rev_id   = rev_id + 1
        version  = trim(row[1])
        pub_date = date2str(row[2])
        initials = trim(row[3])
        notes    = trim(row[4])

        sql = 'INSERT INTO document_rev(rev_id, doc_id, version,'
        sql += ' pub_date, initials, notes)'
        sql += ' VALUES(' + str(rev_id) + ', ' + str(doc_id) + ', ' + wsq(version) + ', '
        sql += wsq(pub_date) + ', ' + wsq(initials) + ', ' + wsq(notes) + ')'
        lampadas_db.runsql(sql)
        lampadas_db.commit()

def copy_document_files():
    sql = 'SELECT doc_id, filename, dtd, format, class FROM document'
    cursor = ldp_db.select(sql)
    while(1):   
        row = cursor.fetchone()
        if row==None: break
        # adjust doc_id to lie above any existing documents
        doc_id   = row[0] + doc_id_offset

        # Ignore child data if parent was ignored.
        if doc_id in ignore_docs:
            continue
        
        filename = trim(row[1])

        # Skip any empty filenames.
        if filename=='':
            continue

        # Guess the CVS directory based on document's attributes
        dtd    = trim(row[2])
        format = trim(row[3])
        type   = trim(row[4])

        # Preface with the dtd or format
        if dtd=='DocBook':   filename = 'docbook/' + filename
        if dtd=='LinuxDoc':  filename = 'linuxdoc/' + filename
        if format=='WIKI':      filename = 'wikitext/' + filename

        # Preface with the type
        if type=='FAQ':      filename = 'faq/' + filename
        if type=='GUIDE':    filename = 'guide/' + filename
        if type=='HOWTO':    filename = 'howto/' + filename
        if type=='MINI':     filename = 'howto/' + filename
        if type=='QUICK':    filename = 'ref/' + filename
        if type=='TEMPLATE': filename = 'howto/' + filename
        
        create_source_file(filename)
            
        sql = 'INSERT INTO document_file(doc_id, filename, top)'
        sql += ' VALUES(' + str(doc_id) + ', ' + wsq(filename) + ', ' + wsq('t') + ')'
        if DEBUG > 0:
            print sql
        lampadas_db.runsql(sql)
        lampadas_db.commit()

def insert_document_files():
    # Manually insert additional files for the demo site.
    insert_document_file(121, 'guide/docbook/LDP-Author-Guide/conventions.xml', 'f')
    insert_document_file(121, 'guide/docbook/LDP-Author-Guide/docbook-xml.xml', 'f')
    insert_document_file(121, 'guide/docbook/LDP-Author-Guide/cvs.xml', 'f')
    insert_document_file(121, 'guide/docbook/LDP-Author-Guide/using-docbook.xml', 'f')
    insert_document_file(121, 'guide/docbook/LDP-Author-Guide/style-guide.xml', 'f')
    insert_document_file(121, 'guide/docbook/LDP-Author-Guide/using-ldp-dsssl.xml', 'f')
    insert_document_file(121, 'guide/docbook/LDP-Author-Guide/using-ldp-xsl.xml', 'f')
    insert_document_file(121, 'guide/docbook/LDP-Author-Guide/fdl-appendix.xml', 'f')
    insert_document_file(121, 'guide/docbook/LDP-Author-Guide/glossary.xml', 'f')
    insert_document_file(121, 'guide/docbook/LDP-Author-Guide/index.xml', 'f')

def insert_document_file(doc_id, filename, top):
    create_source_file(filename)
    sql = 'INSERT INTO document_file(doc_id, filename, top)'
    sql += ' VALUES(' + str(doc_id) + ', ' + wsq(filename) + ', ' + wsq(top) + ')'
    lampadas_db.runsql(sql)
    lampadas_db.commit()

def create_source_file(filename):
    # Create the file if it doesn't already exist
    sql = 'SELECT COUNT(*) FROM sourcefile WHERE filename=' + wsq(filename)
    cursor = lampadas_db.select(sql)
    row = cursor.fetchone()
    if row[0]==0:
        sql = 'INSERT INTO sourcefile(filename) VALUES (' + wsq(filename) + ')'
        lampadas_db.runsql(sql)
        lampadas_db.commit()


def copy_notes():
    note_id = lampadas_db.max_id('document_notes', 'note_id')
    sql = 'SELECT doc_id, date_entered, notes, username FROM notes'
    cursor = ldp_db.select(sql)
    while(1):   
        row = cursor.fetchone()
        if row==None: break
        # adjust doc_id to lie above any existing documents
        doc_id       = row[0] + doc_id_offset

        # Ignore child data if parent was ignored.
        if doc_id in ignore_docs:
            continue

        note_id = note_id + 1
        date_entered = time2str(row[1])
        notes        = trim(row[2])
        username     = trim(row[3])
        
        sql = 'INSERT INTO document_notes(note_id, doc_id, created, '
        sql += ' notes, creator)'
        sql += ' VALUES(' + str(note_id) + ', ' + str(doc_id) + ', ' + wsq(date_entered) + ', '
        sql += wsq(notes) + ', ' + wsq(username) + ')'
        lampadas_db.runsql(sql)
        lampadas_db.commit()


def update_globals():
    sql = 'UPDATE string_i18n SET string=' + wsq('The LDP') + ' WHERE string_code=' + wsq('strprojectshort')
    lampadas_db.runsql(sql)
    lampadas_db.commit()
    
    sql = 'UPDATE string_i18n SET string=' + wsq('The Linux Documentation Project') + ' WHERE string_code=' + wsq('strproject')
    lampadas_db.runsql(sql)
    lampadas_db.commit()


def usage():
    print """Usage: ldp-import [FROM] [FROMHOST] [TO] [TOHOST]

    FROM and TO are the names of two local databases.
    FROMHOST and TOHOST are their hostnames.

    FROM is an LDPDB database, and TO is a Lampadas database.
    """
    sys.exit()


# Options passed on the command line
if len(sys.argv) <> 5:
    usage()

ldp_db_name      = sys.argv[1]
ldp_db_host      = sys.argv[2]
lampadas_db_name = sys.argv[3]
lampadas_db_host = sys.argv[4]

# Databas objects,
ldp_db = get_database('pgsql', ldp_db_name, ldp_db_host)
lampadas_db = get_database('pgsql', lampadas_db_name, lampadas_db_host)

# Collection objects to store user data while merging.
maintainers = OldMaintainers()
maintainer_ids = maintainers.keys()
editors = OldEditors()
editor_ids = editors.keys()
users = OldUsers()
usernames = users.keys()
newusers = NewUsers()
newusernames = newusers.keys()

# Doc ids in the ldp database are all bumped by this much to avoid
# collissions with existing docs in the database. This lets us
# load the LDP data on top of any existing data.
doc_id_offset = lampadas_db.max_id('document', 'doc_id')

# Store docs which were not imported here. Docs with a pub_status of
# "O" (Offsite) are not imported. When loading dependent tables,
# check here to see whether to just ignore the record.
ignore_docs = [] 

# Load the data
print 'Loading LDPDB data...'
copy_users()
copy_documents()
copy_document_users()
copy_document_topics()
copy_document_revs()
copy_document_files()
copy_notes()
update_globals()