View source for Updatelibrary.py

A python script for maintaining a server inventory library and associated asset set.  See [[Custom Libraries]] for more information.

You can download the current version from [http://www.sonic.net/~rknop/php/vw/updatelibrary.py http://www.sonic.net/~rknop/php/vw/updatelibrary.py]

<pre>
#!/usr/bin/python

# updatelibrary.py Version 1.0 by Rob Knop aka Prospero Frobozz (rknop@pobox.com)

# Written with some reference to genassets.pl by Illuminous Beltran/IBM,
# as found on the OpenSim wiki.

# Copyright 2011 Rob Knop (aka Prospero Frobozz). All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#    1. Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.

#    2. Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY ROB KNOP ``AS IS'' AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL ROB KNOP OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and documentation
# are those of the authors and should not be interpreted as representing
# official policies, either expressed or implied, of Rob Knop.


import os, os.path, sys, stat, shutil, re, uuid
import xml.dom, xml.dom.minidom
import traceback
from optparse import OptionParser


opensim_library_root_uuid = "00000112-000f-0000-0000-000100bba000"

# **********************************************************************

class AssetExistsException(Exception):
    def __init__(self, value):
        self.value = value
    def __str__(self, value):
        return repr(value)

class InventoryException(Exception):
    def __init__(self, value):
        self.value = value
    def __str__(self, value):
        return repr(value)

class NiniException(Exception):
    def __init__(self, value):
        self.value = value
    def __str__(self, value):
        return repr(value)

# **********************************************************************

class NiniThing(object):
    def __init__(self, keynames, ninitype):
        self.keynames = keynames
        self.ninitype = ninitype

    def readXML(self, filename):
        sections = []

        if not os.path.exists(filename):
            sys.stderr.write("WARNING: {0} file {1} doesn't exist. Returning empty list.\n".
                             format(self.ninitype, filename))
            return sections

        xmlblob = xml.dom.minidom.parse(filename)
        topelem = xmlblob.documentElement
        if topelem.tagName != "Nini":
            raise NiniException("Error not a Nini file: {0}".format(filename))

        for sec in topelem.childNodes:
            if sec.nodeType == xml.dom.Node.ELEMENT_NODE:
                if sec.tagName != 'Section':
                    raise NiniException("Error {0} file: expecting a 'Section', found a {1}".
                                        format(self.ninitype), sec.tagName)

                item = dict()
                for keyname in self.keynames:
                    item[keyname] = ""

                for seckey in sec.childNodes:
                    if seckey.nodeType == xml.dom.Node.ELEMENT_NODE:
                        if seckey.tagName != "Key":
                            raise NiniException("Error reading {0}: expecting a 'Key', found a {1}".
                                                format(self.ninitype, seckey.tagName) )
                        item[seckey.getAttribute("Name")] = seckey.getAttribute("Value")

                sections.append(item)

        xmlblob.unlink()

        return sections

    def writeXML(self, filename, sections):
        ofh = open(filename, "w")

        ofh.write("<Nini>\n")

        for item in sections:
            ofh.write("  <Section Name=\"{0}\">\n".format(item["name"]))
            for keyname in self.keynames:
                ofh.write("    <Key Name=\"{0}\" Value=\"{1}\" />\n".format(keyname, item[keyname]))
            ofh.write("  </Section>\n\n")

        ofh.write("</Nini>\n")
        ofh.close()


# **********************************************************************

class AssetSet(NiniThing):
    def __init__(self):
        super(AssetSet,self).__init__( [ "assetID",
                                          "name",
                                          "assetType",
                                          "inventoryType",
                                          "fileName"
                                          ],
                                        "Asset Set" )
        self.assets = []

    def __iter__(self):
        return self.assets.__iter__()

    def addasset(self, uuid, name, asstype, invtype, filename):
        ass = { 'assetID': uuid,
                'name': name,
                'assetType': asstype,
                'inventoryType': invtype,
                'fileName': filename }
        self.assets.append(ass)
        return ass

    def findbyfilename(self, filename):
        for ass in self.assets:
            if ass['fileName'] == filename:
                return ass
        return None

    def findbyuuid(self, uuid):
        for ass in self.assets:
            if ass['assetID'] == uuid:
                return ass
        return None

    def readXML(self, filename):
        self.assets = super(AssetSet, self).readXML(filename)

    def writeXML(self, filename):
        super(AssetSet, self).writeXML(filename, self.assets)

# **********************************************************************

class InvFolders(NiniThing):
    def __init__(self):
        super(InvFolders, self).__init__( [ "folderID",
                                            "parentFolderID",
                                            "name",
                                            "type"
                                            ],
                                          "Inventory Folders" )
        self.folders = []

    def __iter__(self):
        return self.folders.__iter__()

    def findbyuuid(self, uuid):
        for folder in self.folders:
            if folder['folderID'] == uuid:
                return folder
        return None

    # NOTE -- findbyname only finds the first folder of
    #  the given name inside the given parent folder.
    #  It is possible, if perverse, to have more than
    #  one folder with the same name...

    def findbyname(self, name, parentid):
        for folder in self.folders:
            if ( ( folder['parentFolderID'] == parentid ) and
                 ( folder['name'] == name ) ):
                return folder
        return None

    def ensureexists(self, name, parentid):
        if ( ( parentid != opensim_library_root_uuid ) and
             ( not self.findbyuuid(parentid) ) ):
            raise InventoryException("No such parent folder exists: " + str(parentid))

        folder = self.findbyname(name, parentid)
        if folder != None:
            return folder

        # HACK ALERT -- I want this "type 6" to become whatever the right
        #  type is for no icon
        # http://opensimulator.org/wiki/AssetServer/DeveloperDocs doesn't tell me enough

        folder = { 'folderID' : uuid.uuid4(),
                   'parentFolderID' : parentid,
                   'name' : name,
                   'type' : 6
                  }
        self.folders.append(folder)

        return folder


    def readXML(self, filename):
        self.folders = super(InvFolders, self).readXML(filename)

    def writeXML(self, filename):
        super(InvFolders, self).writeXML(filename, self.folders)



# **********************************************************************

class InvItems(NiniThing):
    def __init__(self, folders):
        super(InvItems, self).__init__( [ "inventoryID",
                                          "assetID",
                                          "folderID",
                                          "name",
                                          "description",
                                          "assetType",
                                          "inventoryType",
                                          "currentPermissions",
                                          "nextPermissions",
                                          "everyonePermissions",
                                          "basePermissions"
                                          ],
                                        "Inventory Items" )
        self.folders = folders
        self.items = []

    def findbyinvid(self, uuid):
        for item in self.items:
            if item['inventoryID'] == uuid:
                return item
        return None

    # Note -- findbyname only finds the first one of that name
    #  in that folder.  You can have more than one thing of
    #  the same name in a given folder....

    def findbyname(self, name, folderid):
        for item in self.items:
            if ( ( item['name'] == name ) and
                 ( item['folderID'] == folderid) ):
               return item
        return None

    def ensureexists(self, name, assetid, assettype, invtype, folderid):
        if ( ( folderid != opensim_library_root_uuid) and
             ( not self.folders.findbyuuid(folderid) ) ):
            raise InventoryException("No such folder for item: " + str(folderid))

        for item in self.items:
            if ( ( item['assetID'] == assetid ) and
                 ( item['folderID'] == folderid ) and
                 ( item['name'] == name ) ):
                return item

        item = { 'inventoryID' : uuid.uuid4(),
                 'assetID' : assetid,
                 'folderID' : folderid,
                 'name' : name,
                 'description' : "",
                 'assetType' : assettype,
                 'inventoryType' : invtype,
                 'currentPermissions' : "2147483647",
                 'nextPermissions' : "2147483647",
                 'everyonePermissions' : "2147483647",
                 'basePermissions' : "2147483647"
               }
        self.items.append(item)

        return item


    def readXML(self, filename):
        self.items = super(InvItems, self).readXML(filename)

    def writeXML(self, filename):
        super(InvItems, self).writeXML(filename, self.items)


# **********************************************************************

class LibraryGenerator(object):
    def __init__(self, libname, shortname):
        self.libname = libname
        self.shortname = shortname
        self.asset_dir = os.path.abspath(os.path.join("assets", "{0}AssetSet".format(shortname)))
        self.inv_dir = os.path.abspath(os.path.join("inventory", shortname))
        self.asset_xmlfile = "{0}AssetSet.xml".format(shortname)
        self.folders_xmlfile = "{0}InvFolders.xml".format(shortname)
        self.items_xmlfile = "{0}InvItems.xml".format(shortname)
        self.skips = []

        self.matchj2k = re.compile("\.j2k$")
        self.matchtxt = re.compile("\.txt$")
        self.matchlsl = re.compile("\.lsl$")
        self.endsinslash = re.compile("/$")
        self.commentline = re.compile("^\s*#")
        self.blankline = re.compile("^\s*$")
        self.assetlistline = re.compile("^\s*([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\s*(\d+)\s*(\d+)\s*(.*\S)\s*$")

    def set_skip_patterns(self, skips):
        self.skips = skips

    def set_asset_dir(self, asset_dir):
        self.asset_dir = asset_dir

    def set_inv_dir(self, inv_dir):
        self.inv_dir = inv_dir

    def set_asset_xmlfile(self, asset_xmlfile):
        self.asset_xmlfile = asset_xmlfile

    def set_folders_xmlfile(self, folders_xmlfile):
        self.folders_xmlfile = folders_xmlfile

    def set_items_xmlfile(self, items_xmlfile):
        self.items_xmlfile = items_xmlfile


    def run(self):
        # Create our storage for things, and read any current
        #  XML files if they exist

        self.assets = AssetSet()
        self.assets.readXML(os.path.join(self.asset_dir, self.asset_xmlfile))

        self.folders = InvFolders()
        self.folders.readXML(os.path.join(self.inv_dir, self.folders_xmlfile))

        self.items = InvItems(self.folders)
        self.items.readXML(os.path.join(self.inv_dir, self.items_xmlfile))


        # Run the recursive directory processor thingie

        origdir = os.getcwd()
        os.chdir(self.asset_dir)
        self.process_dir(opensim_library_root_uuid, "", "")
        os.chdir(origdir)

        # Write the XML files, saving copies of the old ones out of paranoia

        if os.path.exists(os.path.join(self.asset_dir, self.asset_xmlfile)):
            shutil.copy2(os.path.join(self.asset_dir, self.asset_xmlfile),
                         os.path.join(self.asset_dir, "{0}-old".format(self.asset_xmlfile)))

        self.assets.writeXML(os.path.join(self.asset_dir, self.asset_xmlfile))

        if os.path.exists(os.path.join(self.inv_dir, self.folders_xmlfile)):
            shutil.copy2(os.path.join(self.inv_dir, self.folders_xmlfile),
                         os.path.join(self.inv_dir, "{0}-old".format(self.folders_xmlfile)))

        self.folders.writeXML(os.path.join(self.inv_dir, self.folders_xmlfile))

        if os.path.exists(os.path.join(self.inv_dir, self.items_xmlfile)):
            shutil.copy2(os.path.join(self.inv_dir, self.items_xmlfile),
                         os.path.join(self.inv_dir, "{0}-old".format(self.items_xmlfile)))

        self.items.writeXML(os.path.join(self.inv_dir, self.items_xmlfile))


    def process_dir(self, parent_uuid, my_name, path):
        origdir = os.getcwd()
        sys.stderr.write("Processing directory: {0} \n".format(os.path.join(self.asset_dir, path)))
        if my_name != "":
            os.chdir(my_name)
        else:
            my_name = self.libname
            
        # Create the folder XML blob if necessary

        curfolder = self.folders.ensureexists(my_name, parent_uuid)
        my_uuid = curfolder["folderID"]

        # Walk through this directory and process everything in it

        for fname in os.listdir("."):
            statinfo = os.stat(fname)

            doskip = False
            for skip in self.skips:
                if re.search(skip, fname):
                    doskip = True
            if doskip: continue

            if stat.S_ISDIR(statinfo.st_mode):
                self.process_dir(my_uuid, fname, os.path.join(path, fname) )
                continue

            fpath = os.path.join(path, fname)
            
            if fpath == self.asset_xmlfile:
                # Ignore the XML file we're working on!
                continue

            elif fname == "addassets.lis":
                # Ignore the file specifing pre-existing assets to add
                #   to inventory
                continue

            elif self.matchj2k.search(fname):
                asset_type = 0
            elif self.matchtxt.search(fname):
                asset_type = 7
            elif self.matchlsl.search(fname):
                asset_type = 10
            else:
                sys.stderr.write("WARNING: Skipping unrecognized file " +
                                 fname + "\n")
                continue

            # Strip the extension for aesthetic purposes
            #  (Right now, that must be .txt, .lsl, or .j2k, so
            #   we can assume it's 4 characters long.)

            name = fname[0:-4]

            # Create the asset XML blob if necessary

            ass = self.assets.findbyfilename(fpath)
            if ass == None:
                ass = self.assets.addasset(uuid.uuid4(), name, asset_type,
                                           asset_type, os.path.join(path, fname))

            # Create the inventory XML blob if necessary

            self.items.ensureexists(name, ass['assetID'], asset_type, asset_type, my_uuid)

        # Add pre-existing assets (to inventory only) from file addassets.lis

        if os.path.exists("addassets.lis"):
            asslis = open("addassets.lis")
            for line in asslis:
                line.rstrip("\r\n")
                if self.blankline.search(line) or self.commentline.search(line):
                    continue
                match = self.assetlistline.search(line)
                if match == None:
                    sys.stderr.write("WARNING : error parsing line in {0}\n".
                                     format(os.path.join(path, "addassets.lis")))
                    sys.stderr.write("Offending line: \"{0}\"\n".format(line))
                    continue
                self.items.ensureexists(match.group(4), match.group(1), match.group(2), match.group(3), my_uuid)
            asslis.close()

        # We're done; restore the CWD to where we started and return

        os.chdir(origdir)

# **********************************************************************

def main():

    usage = """
    updatelibrary.py -n "Library Name" -s ShortLibName [options]

  Goes through your asset set directory.  (See below for how to specify
  it, and what the default is.)  Finds all .j2k images, .txt files, and
  .lsl files in that directory and all subdirectories.  Generates an
  Asset Set XML file for all of those files.  Also generates library
  Inventory Folder and Inventory Item XML files for all of those assets,
  organized in a folder hierarchy that matches the directory hierarchy.

  Additional assets that already exist in the grid's asset store may be
  specified in each directory in a file named "addassets.lis".

  For more documentation, see http://opensimulator.org/wiki/Custom_Libraries

  When done, you will need to manually add your asset directory to the
  OpenSim AssetSets.xml file (in "assets" under the OpenSim bin
  directory), and your inventory XML files to the Libraries.xml file (in
  "inventory" under the OpenSim bin directory).

  Written 2011 by Rob Knop aka Prospero Frobozz.  I've only used this on
  Linux; I have no idea how well it will function on Windows.  If you're
  on Windows, I recommend http://www.ubuntu.com

  Run updatelibrary.py --help for option listings
"""

    parser = OptionParser(usage = usage)

    parser.add_option("-n", "--lib-name", action="store", type="string",
                      dest="libname", default="New Library",
                      help="The name of your library as it appears in the viewer")

    parser.add_option("-s", "--short-name", action="store", type="string",
                      dest="shortname", default="NewLibrary",
                      help="A name without spaces of your library for filename purposes.")

    parser.add_option("-a", "--asset-dir", action="store",
                      type="string", dest="asset_dir",
                      default=None,
                      help="Directory where your .j2k, .txt, and .lsl files are, and which will "+
                      "serve as the Assset Set in the OpenSim library.  Defaults to assets/[shortname]AssetSet")

    parser.add_option("-i", "--inv-dir", action="store", type="string",
                      dest="inv_dir", default=None,
                      help="Directory in which to put the inventory XML files.  Defaults to "+
                      "inventory/[shortname]")

    parser.add_option("", "--asset-xmlfile", action="store",
                      type="string", dest="asset_xmlfile", default=None,
                      help="Name of Asset XML file.  Defaults to [shortname]AssetSet.xml")

    parser.add_option("", "--folders-xmlfile", action="store",
                      type="string", dest="folders_xmlfile", default=None,
                      help="Name of Inventory Folders XML file.  Defaults to [shortname]InvFolders.xml")

    parser.add_option("", "--items-xmlfile", action="store",
                      type="string", dest="items_xmlfile", default=None,
                      help="Name of Inventory Folders XML file.  Defaults to [shortname]InvItems.xml")

    parser.add_option("", "--skip", action="append", type="string",
                      dest="skip_patterns",
                      default=['^\.hg$', '^\.git$', '^\.svn$'],
                      help="Regex for filenames to not process")

    if len(sys.argv) < 2:
        parser.print_usage()
        sys.exit(0)

    options, args = parser.parse_args()

    try:
        generator = LibraryGenerator(options.libname, options.shortname)

        if options.skip_patterns:
            generator.set_skip_patterns(options.skip_patterns)

        if options.asset_dir != None:
            generator.set_asset_dir(options.asset_dir)

        if options.inv_dir != None:
            generator.set_inv_dir(options.inv_dir)

        if options.asset_xmlfile != None:
            generator.set_asset_xmlfile(options.asset_xmlfile)

        if options.folders_xmlfile != None:
            generator.set_folders_xmlfile(options.folders_xmlfile)

        if options.items_xmlfile != None:
            generator.set_items_xmlfile(options.items_xmlfile)

        generator.run()

    except Exception:
        print "FAIL"
        traceback.print_exc()
        sys.exit(20)


# **********************************************************************

if __name__ == "__main__":
    sys.exit(main())
</pre>