kc3-lang/libxml2/doc/index.py

• Show log

  Commit

• Hash : 24f6a071
  Author :
  Date : 2004-04-08T14:39:25
  

  Some preliminary stats for queries extraction, Daniel
  

Download

• doc/index.py

• #!/usr/bin/python -u
  #
  # imports the API description and fills up a database with
  # name relevance to modules, functions or web pages
  #
  # Operation needed:
  # =================
  #
  # install mysqld, the python wrappers for mysql and libxml2, start mysqld
  # Change the root passwd of mysql:
  #    mysqladmin -u root password new_password
  # Create the new database xmlsoft
  #    mysqladmin -p create xmlsoft
  # Create a database user 'veillard' and give him passord access
  # change veillard and abcde with the right user name and passwd
  #    mysql -p
  #    password:
  #    mysql> GRANT ALL PRIVILEGES ON xmlsoft TO veillard@localhost
  #           IDENTIFIED BY 'abcde' WITH GRANT OPTION;
  #
  # As the user check the access:
  #    mysql -p xmlsoft
  #    Enter password:
  #    Welcome to the MySQL monitor....
  #    mysql> use xmlsoft
  #    Database changed
  #    mysql> quit
  #    Bye
  #
  # Then run the script in the doc subdir, it will create the symbols and
  # word tables and populate them with informations extracted from 
  # the libxml2-api.xml API description, and make then accessible read-only
  # by nobody@loaclhost the user expected to be Apache's one
  #
  # On the Apache configuration, make sure you have php support enabled
  #
  
  import MySQLdb
  import libxml2
  import sys
  import string
  import os
  
  #
  # We are not interested in parsing errors here
  #
  def callback(ctx, str):
      return
  libxml2.registerErrorHandler(callback, None)
  
  #
  # The dictionnary of tables required and the SQL command needed
  # to create them
  #
  TABLES={
    "symbols" : """CREATE TABLE symbols (
             name varchar(255) BINARY NOT NULL,
  	   module varchar(255) BINARY NOT NULL,
             type varchar(25) NOT NULL,
  	   descr varchar(255),
  	   UNIQUE KEY name (name),
  	   KEY module (module))""",
    "words" : """CREATE TABLE words (
             name varchar(50) BINARY NOT NULL,
  	   symbol varchar(255) BINARY NOT NULL,
             relevance int,
  	   KEY name (name),
  	   KEY symbol (symbol),
  	   UNIQUE KEY ID (name, symbol))""",
    "wordsHTML" : """CREATE TABLE wordsHTML (
             name varchar(50) BINARY NOT NULL,
  	   resource varchar(255) BINARY NOT NULL,
  	   section varchar(255),
  	   id varchar(50),
             relevance int,
  	   KEY name (name),
  	   KEY resource (resource),
  	   UNIQUE KEY ref (name, resource))""",
    "wordsArchive" : """CREATE TABLE wordsArchive (
             name varchar(50) BINARY NOT NULL,
  	   ID int(11) NOT NULL,
             relevance int,
  	   KEY name (name),
  	   UNIQUE KEY ref (name, ID))""",
    "pages" : """CREATE TABLE pages (
             resource varchar(255) BINARY NOT NULL,
  	   title varchar(255) BINARY NOT NULL,
  	   UNIQUE KEY name (resource))""",
    "archives" : """CREATE TABLE archives (
             ID int(11) NOT NULL auto_increment,
             resource varchar(255) BINARY NOT NULL,
  	   title varchar(255) BINARY NOT NULL,
  	   UNIQUE KEY id (ID,resource(255)),
  	   INDEX (ID),
  	   INDEX (resource))""",
    "Queries" : """CREATE TABLE Queries (
             ID int(11) NOT NULL auto_increment,
  	   Value varchar(50) NOT NULL,
  	   Count int(11) NOT NULL,
  	   UNIQUE KEY id (ID,Value(35)),
  	   INDEX (ID))""",
    "AllQueries" : """CREATE TABLE AllQueries (
             ID int(11) NOT NULL auto_increment,
  	   Value varchar(50) NOT NULL,
  	   Count int(11) NOT NULL,
  	   UNIQUE KEY id (ID,Value(35)),
  	   INDEX (ID))""",
  }
  
  #
  # The XML API description file to parse
  #
  API="libxml2-api.xml"
  DB=None
  
  #########################################################################
  #									#
  #                  MySQL database interfaces				#
  #									#
  #########################################################################
  def createTable(db, name):
      global TABLES
  
      if db == None:
          return -1
      if name == None:
          return -1
      c = db.cursor()
  
      ret = c.execute("DROP TABLE IF EXISTS %s" % (name))
      if ret == 1:
          print "Removed table %s" % (name)
      print "Creating table %s" % (name)
      try:
          ret = c.execute(TABLES[name])
      except:
          print "Failed to create table %s" % (name)
  	return -1
      return ret
  
  def checkTables(db, verbose = 1):
      global TABLES
  
      if db == None:
          return -1
      c = db.cursor()
      nbtables = c.execute("show tables")
      if verbose:
  	print "Found %d tables" % (nbtables)
      tables = {}
      i = 0
      while i < nbtables:
          l = c.fetchone()
  	name = l[0]
  	tables[name] = {}
          i = i + 1
  
      for table in TABLES.keys():
          if not tables.has_key(table):
  	    print "table %s missing" % (table)
  	    createTable(db, table)
  	try:
  	    ret = c.execute("SELECT count(*) from %s" % table);
  	    row = c.fetchone()
  	    if verbose:
  		print "Table %s contains %d records" % (table, row[0])
  	except:
  	    print "Troubles with table %s : repairing" % (table)
  	    ret = c.execute("repair table %s" % table);
  	    print "repairing returned %d" % (ret)
  	    ret = c.execute("SELECT count(*) from %s" % table);
  	    row = c.fetchone()
  	    print "Table %s contains %d records" % (table, row[0])
      if verbose:
  	print "checkTables finished"
  
      # make sure apache can access the tables read-only
      try:
  	ret = c.execute("GRANT SELECT ON xmlsoft.* TO nobody@localhost")
  	ret = c.execute("GRANT INSERT,SELECT,UPDATE  ON xmlsoft.Queries TO nobody@localhost")
      except:
          pass
      return 0
      
  def openMySQL(db="xmlsoft", passwd=None, verbose = 1):
      global DB
  
      if passwd == None:
          try:
  	    passwd = os.environ["MySQL_PASS"]
  	except:
  	    print "No password available, set environment MySQL_PASS"
  	    sys.exit(1)
  
      DB = MySQLdb.connect(passwd=passwd, db=db)
      if DB == None:
          return -1
      ret = checkTables(DB, verbose)
      return ret
  
  def updateWord(name, symbol, relevance):
      global DB
  
      if DB == None:
          openMySQL()
      if DB == None:
          return -1
      if name == None:
          return -1
      if symbol == None:
          return -1
  
      c = DB.cursor()
      try:
  	ret = c.execute(
  """INSERT INTO words (name, symbol, relevance) VALUES ('%s','%s', %d)""" %
  		(name, symbol, relevance))
      except:
          try:
  	    ret = c.execute(
      """UPDATE words SET relevance = %d where name = '%s' and symbol = '%s'""" %
  		    (relevance, name, symbol))
  	except:
  	    print "Update word (%s, %s, %s) failed command" % (name, symbol, relevance)
  	    print "UPDATE words SET relevance = %d where name = '%s' and symbol = '%s'" % (relevance, name, symbol)
  	    print sys.exc_type, sys.exc_value
  	    return -1
  	     
      return ret
  
  def updateSymbol(name, module, type, desc):
      global DB
  
      updateWord(name, name, 50)
      if DB == None:
          openMySQL()
      if DB == None:
          return -1
      if name == None:
          return -1
      if module == None:
          return -1
      if type == None:
          return -1
  
      try:
  	desc = string.replace(desc, "'", " ")
  	l = string.split(desc, ".")
  	desc = l[0]
  	desc = desc[0:99]
      except:
          desc = ""
  
      c = DB.cursor()
      try:
  	ret = c.execute(
  """INSERT INTO symbols (name, module, type, descr) VALUES ('%s','%s', '%s', '%s')""" %
                      (name, module, type, desc))
      except:
          try:
  	    ret = c.execute(
  """UPDATE symbols SET module='%s', type='%s', descr='%s' where name='%s'""" %
                      (module, type, desc, name))
          except:
  	    print "Update symbol (%s, %s, %s) failed command" % (name, module, type)
  	    print """UPDATE symbols SET module='%s', type='%s', descr='%s' where name='%s'""" % (module, type, desc, name)
  	    print sys.exc_type, sys.exc_value
  	    return -1
  	     
      return ret
          
  def addFunction(name, module, desc = ""):
      return updateSymbol(name, module, 'function', desc)
  
  def addMacro(name, module, desc = ""):
      return updateSymbol(name, module, 'macro', desc)
  
  def addEnum(name, module, desc = ""):
      return updateSymbol(name, module, 'enum', desc)
  
  def addStruct(name, module, desc = ""):
      return updateSymbol(name, module, 'struct', desc)
  
  def addConst(name, module, desc = ""):
      return updateSymbol(name, module, 'const', desc)
  
  def addType(name, module, desc = ""):
      return updateSymbol(name, module, 'type', desc)
  
  def addFunctype(name, module, desc = ""):
      return updateSymbol(name, module, 'functype', desc)
  
  def addPage(resource, title):
      global DB
  
      if DB == None:
          openMySQL()
      if DB == None:
          return -1
      if resource == None:
          return -1
  
      c = DB.cursor()
      try:
  	ret = c.execute(
  	    """INSERT INTO pages (resource, title) VALUES ('%s','%s')""" %
                      (resource, title))
      except:
          try:
  	    ret = c.execute(
  		"""UPDATE pages SET title='%s' WHERE resource='%s'""" %
                      (title, resource))
          except:
  	    print "Update symbol (%s, %s, %s) failed command" % (name, module, type)
  	    print """UPDATE pages SET title='%s' WHERE resource='%s'""" % (title, resource)
  	    print sys.exc_type, sys.exc_value
  	    return -1
  	     
      return ret
  
  def updateWordHTML(name, resource, desc, id, relevance):
      global DB
  
      if DB == None:
          openMySQL()
      if DB == None:
          return -1
      if name == None:
          return -1
      if resource == None:
          return -1
      if id == None:
          id = ""
      if desc == None:
          desc = ""
      else:
  	try:
  	    desc = string.replace(desc, "'", " ")
  	    desc = desc[0:99]
  	except:
  	    desc = ""
  
      c = DB.cursor()
      try:
  	ret = c.execute(
  """INSERT INTO wordsHTML (name, resource, section, id, relevance) VALUES ('%s','%s', '%s', '%s', '%d')""" %
                      (name, resource, desc, id, relevance))
      except:
          try:
  	    ret = c.execute(
  """UPDATE wordsHTML SET section='%s', id='%s', relevance='%d' where name='%s' and resource='%s'""" %
                      (desc, id, relevance, name, resource))
          except:
  	    print "Update symbol (%s, %s, %d) failed command" % (name, resource, relevance)
  	    print """UPDATE wordsHTML SET section='%s', id='%s', relevance='%d' where name='%s' and resource='%s'""" % (desc, id, relevance, name, resource)
  	    print sys.exc_type, sys.exc_value
  	    return -1
  	     
      return ret
  
  def checkXMLMsgArchive(url):
      global DB
  
      if DB == None:
          openMySQL()
      if DB == None:
          return -1
      if url == None:
          return -1
  
      c = DB.cursor()
      try:
  	ret = c.execute(
  	    """SELECT ID FROM archives WHERE resource='%s'""" % (url))
  	row = c.fetchone()
  	if row == None:
  	    return -1
      except:
  	return -1
  	     
      return row[0]
      
  def addXMLMsgArchive(url, title):
      global DB
  
      if DB == None:
          openMySQL()
      if DB == None:
          return -1
      if url == None:
          return -1
      if title == None:
          title = ""
      else:
  	title = string.replace(title, "'", " ")
  	title = title[0:99]
  
      c = DB.cursor()
      try:
          cmd = """INSERT INTO archives (resource, title) VALUES ('%s','%s')""" % (url, title)
          ret = c.execute(cmd)
  	cmd = """SELECT ID FROM archives WHERE resource='%s'""" % (url)
          ret = c.execute(cmd)
  	row = c.fetchone()
  	if row == None:
  	    print "addXMLMsgArchive failed to get the ID: %s" % (url)
  	    return -1
      except:
          print "addXMLMsgArchive failed command: %s" % (cmd)
  	return -1
  	     
      return((int)(row[0]))
  
  def updateWordArchive(name, id, relevance):
      global DB
  
      if DB == None:
          openMySQL()
      if DB == None:
          return -1
      if name == None:
          return -1
      if id == None:
          return -1
  
      c = DB.cursor()
      try:
  	ret = c.execute(
  """INSERT INTO wordsArchive (name, id, relevance) VALUES ('%s', '%d', '%d')""" %
                      (name, id, relevance))
      except:
          try:
  	    ret = c.execute(
  """UPDATE wordsArchive SET relevance='%d' where name='%s' and ID='%d'""" %
                      (relevance, name, id))
          except:
  	    print "Update word archive (%s, %d, %d) failed command" % (name, id, relevance)
  	    print """UPDATE wordsArchive SET relevance='%d' where name='%s' and ID='%d'""" % (relevance, name, id)
  	    print sys.exc_type, sys.exc_value
  	    return -1
  	     
      return ret
  
  #########################################################################
  #									#
  #                  Word dictionnary and analysis routines		#
  #									#
  #########################################################################
  
  #
  # top 100 english word without the one len < 3 + own set
  #
  dropWords = {
      'the':0, 'this':0, 'can':0, 'man':0, 'had':0, 'him':0, 'only':0,
      'and':0, 'not':0, 'been':0, 'other':0, 'even':0, 'are':0, 'was':0,
      'new':0, 'most':0, 'but':0, 'when':0, 'some':0, 'made':0, 'from':0,
      'who':0, 'could':0, 'after':0, 'that':0, 'will':0, 'time':0, 'also':0,
      'have':0, 'more':0, 'these':0, 'did':0, 'was':0, 'two':0, 'many':0,
      'they':0, 'may':0, 'before':0, 'for':0, 'which':0, 'out':0, 'then':0,
      'must':0, 'one':0, 'through':0, 'with':0, 'you':0, 'said':0,
      'first':0, 'back':0, 'were':0, 'what':0, 'any':0, 'years':0, 'his':0,
      'her':0, 'where':0, 'all':0, 'its':0, 'now':0, 'much':0, 'she':0,
      'about':0, 'such':0, 'your':0, 'there':0, 'into':0, 'like':0, 'may':0,
      'would':0, 'than':0, 'our':0, 'well':0, 'their':0, 'them':0, 'over':0,
      'down':0,
      'net':0, 'www':0, 'bad':0, 'Okay':0, 'bin':0, 'cur':0,
  }
  
  wordsDict = {}
  wordsDictHTML = {}
  wordsDictArchive = {}
  
  def cleanupWordsString(str):
      str = string.replace(str, ".", " ")
      str = string.replace(str, "!", " ")
      str = string.replace(str, "?", " ")
      str = string.replace(str, ",", " ")
      str = string.replace(str, "'", " ")
      str = string.replace(str, '"', " ")
      str = string.replace(str, ";", " ")
      str = string.replace(str, "(", " ")
      str = string.replace(str, ")", " ")
      str = string.replace(str, "{", " ")
      str = string.replace(str, "}", " ")
      str = string.replace(str, "<", " ")
      str = string.replace(str, ">", " ")
      str = string.replace(str, "=", " ")
      str = string.replace(str, "/", " ")
      str = string.replace(str, "*", " ")
      str = string.replace(str, ":", " ")
      str = string.replace(str, "#", " ")
      str = string.replace(str, "\\", " ")
      str = string.replace(str, "\n", " ")
      str = string.replace(str, "\r", " ")
      str = string.replace(str, "\xc2", " ")
      str = string.replace(str, "\xa0", " ")
      return str
      
  def cleanupDescrString(str):
      str = string.replace(str, "'", " ")
      str = string.replace(str, "\n", " ")
      str = string.replace(str, "\r", " ")
      str = string.replace(str, "\xc2", " ")
      str = string.replace(str, "\xa0", " ")
      l = string.split(str)
      str = string.join(str)
      return str
  
  def splitIdentifier(str):
      ret = []
      while str != "":
          cur = string.lower(str[0])
  	str = str[1:]
  	if ((cur < 'a') or (cur > 'z')):
  	    continue
  	while (str != "") and (str[0] >= 'A') and (str[0] <= 'Z'):
  	    cur = cur + string.lower(str[0])
  	    str = str[1:]
  	while (str != "") and (str[0] >= 'a') and (str[0] <= 'z'):
  	    cur = cur + str[0]
  	    str = str[1:]
  	while (str != "") and (str[0] >= '0') and (str[0] <= '9'):
  	    str = str[1:]
  	ret.append(cur)
      return ret
  
  def addWord(word, module, symbol, relevance):
      global wordsDict
  
      if word == None or len(word) < 3:
          return -1
      if module == None or symbol == None:
          return -1
      if dropWords.has_key(word):
          return 0
      if ord(word[0]) > 0x80:
          return 0
  
      if wordsDict.has_key(word):
          d = wordsDict[word]
  	if d == None:
  	    return 0
  	if len(d) > 500:
  	    wordsDict[word] = None
  	    return 0
  	try:
  	    relevance = relevance + d[(module, symbol)]
  	except:
  	    pass
      else:
          wordsDict[word] = {}
      wordsDict[word][(module, symbol)] = relevance
      return relevance
      
  def addString(str, module, symbol, relevance):
      if str == None or len(str) < 3:
          return -1
      ret = 0
      str = cleanupWordsString(str)
      l = string.split(str)
      for word in l:
  	if len(word) > 2:
  	    ret = ret + addWord(word, module, symbol, 5)
  
      return ret
  
  def addWordHTML(word, resource, id, section, relevance):
      global wordsDictHTML
  
      if word == None or len(word) < 3:
          return -1
      if resource == None or section == None:
          return -1
      if dropWords.has_key(word):
          return 0
      if ord(word[0]) > 0x80:
          return 0
  
      section = cleanupDescrString(section)
  
      if wordsDictHTML.has_key(word):
          d = wordsDictHTML[word]
  	if d == None:
  	    print "skipped %s" % (word)
  	    return 0
  	try:
  	    (r,i,s) = d[resource]
  	    if i != None:
  	        id = i
  	    if s != None:
  	        section = s
  	    relevance = relevance + r
  	except:
  	    pass
      else:
          wordsDictHTML[word] = {}
      d = wordsDictHTML[word];
      d[resource] = (relevance, id, section)
      return relevance
      
  def addStringHTML(str, resource, id, section, relevance):
      if str == None or len(str) < 3:
          return -1
      ret = 0
      str = cleanupWordsString(str)
      l = string.split(str)
      for word in l:
  	if len(word) > 2:
  	    try:
  		r = addWordHTML(word, resource, id, section, relevance)
  		if r < 0:
  		    print "addWordHTML failed: %s %s" % (word, resource)
  		ret = ret + r
  	    except:
  		print "addWordHTML failed: %s %s %d" % (word, resource, relevance)
  		print sys.exc_type, sys.exc_value
  
      return ret
  
  def addWordArchive(word, id, relevance):
      global wordsDictArchive
  
      if word == None or len(word) < 3:
          return -1
      if id == None or id == -1:
          return -1
      if dropWords.has_key(word):
          return 0
      if ord(word[0]) > 0x80:
          return 0
  
      if wordsDictArchive.has_key(word):
          d = wordsDictArchive[word]
  	if d == None:
  	    print "skipped %s" % (word)
  	    return 0
  	try:
  	    r = d[id]
  	    relevance = relevance + r
  	except:
  	    pass
      else:
          wordsDictArchive[word] = {}
      d = wordsDictArchive[word];
      d[id] = relevance
      return relevance
      
  def addStringArchive(str, id, relevance):
      if str == None or len(str) < 3:
          return -1
      ret = 0
      str = cleanupWordsString(str)
      l = string.split(str)
      for word in l:
          i = len(word)
  	if i > 2:
  	    try:
  		r = addWordArchive(word, id, relevance)
  		if r < 0:
  		    print "addWordArchive failed: %s %s" % (word, id)
  		else:
  		    ret = ret + r
  	    except:
  		print "addWordArchive failed: %s %s %d" % (word, id, relevance)
  		print sys.exc_type, sys.exc_value
      return ret
  
  #########################################################################
  #									#
  #                  XML API description analysis				#
  #									#
  #########################################################################
  
  def loadAPI(filename):
      doc = libxml2.parseFile(filename)
      print "loaded %s" % (filename)
      return doc
  
  def foundExport(file, symbol):
      if file == None:
          return 0
      if symbol == None:
          return 0
      addFunction(symbol, file)
      l = splitIdentifier(symbol)
      for word in l:
  	addWord(word, file, symbol, 10)
      return 1
       
  def analyzeAPIFile(top):
      count = 0
      name = top.prop("name")
      cur = top.children
      while cur != None:
          if cur.type == 'text':
  	    cur = cur.next
  	    continue
  	if cur.name == "exports":
  	    count = count + foundExport(name, cur.prop("symbol"))
  	else:
  	    print "unexpected element %s in API doc <file name='%s'>" % (name)
          cur = cur.next
      return count
  
  def analyzeAPIFiles(top):
      count = 0
      cur = top.children
          
      while cur != None:
          if cur.type == 'text':
  	    cur = cur.next
  	    continue
  	if cur.name == "file":
  	    count = count + analyzeAPIFile(cur)
  	else:
  	    print "unexpected element %s in API doc <files>" % (cur.name)
          cur = cur.next
      return count
  
  def analyzeAPIEnum(top):
      file = top.prop("file")
      if file == None:
          return 0
      symbol = top.prop("name")
      if symbol == None:
          return 0
  
      addEnum(symbol, file)
      l = splitIdentifier(symbol)
      for word in l:
  	addWord(word, file, symbol, 10)
  
      return 1
  
  def analyzeAPIConst(top):
      file = top.prop("file")
      if file == None:
          return 0
      symbol = top.prop("name")
      if symbol == None:
          return 0
  
      addConst(symbol, file)
      l = splitIdentifier(symbol)
      for word in l:
  	addWord(word, file, symbol, 10)
  
      return 1
  
  def analyzeAPIType(top):
      file = top.prop("file")
      if file == None:
          return 0
      symbol = top.prop("name")
      if symbol == None:
          return 0
  
      addType(symbol, file)
      l = splitIdentifier(symbol)
      for word in l:
  	addWord(word, file, symbol, 10)
      return 1
  
  def analyzeAPIFunctype(top):
      file = top.prop("file")
      if file == None:
          return 0
      symbol = top.prop("name")
      if symbol == None:
          return 0
  
      addFunctype(symbol, file)
      l = splitIdentifier(symbol)
      for word in l:
  	addWord(word, file, symbol, 10)
      return 1
  
  def analyzeAPIStruct(top):
      file = top.prop("file")
      if file == None:
          return 0
      symbol = top.prop("name")
      if symbol == None:
          return 0
  
      addStruct(symbol, file)
      l = splitIdentifier(symbol)
      for word in l:
  	addWord(word, file, symbol, 10)
  
      info = top.prop("info")
      if info != None:
  	info = string.replace(info, "'", " ")
  	info = string.strip(info)
  	l = string.split(info)
  	for word in l:
  	    if len(word) > 2:
  		addWord(word, file, symbol, 5)
      return 1
  
  def analyzeAPIMacro(top):
      file = top.prop("file")
      if file == None:
          return 0
      symbol = top.prop("name")
      if symbol == None:
          return 0
      symbol = string.replace(symbol, "'", " ")
      symbol = string.strip(symbol)
  
      info = None
      cur = top.children
      while cur != None:
          if cur.type == 'text':
  	    cur = cur.next
  	    continue
  	if cur.name == "info":
  	    info = cur.content
  	    break
          cur = cur.next
  
      l = splitIdentifier(symbol)
      for word in l:
  	addWord(word, file, symbol, 10)
  
      if info == None:
  	addMacro(symbol, file)
          print "Macro %s description has no <info>" % (symbol)
          return 0
  
      info = string.replace(info, "'", " ")
      info = string.strip(info)
      addMacro(symbol, file, info)
      l = string.split(info)
      for word in l:
  	if len(word) > 2:
  	    addWord(word, file, symbol, 5)
      return 1
  
  def analyzeAPIFunction(top):
      file = top.prop("file")
      if file == None:
          return 0
      symbol = top.prop("name")
      if symbol == None:
          return 0
  
      symbol = string.replace(symbol, "'", " ")
      symbol = string.strip(symbol)
      info = None
      cur = top.children
      while cur != None:
          if cur.type == 'text':
  	    cur = cur.next
  	    continue
  	if cur.name == "info":
  	    info = cur.content
  	elif cur.name == "return":
  	    rinfo = cur.prop("info")
  	    if rinfo != None:
  		rinfo = string.replace(rinfo, "'", " ")
  		rinfo = string.strip(rinfo)
  	        addString(rinfo, file, symbol, 7)
  	elif cur.name == "arg":
  	    ainfo = cur.prop("info")
  	    if ainfo != None:
  		ainfo = string.replace(ainfo, "'", " ")
  		ainfo = string.strip(ainfo)
  	        addString(ainfo, file, symbol, 5)
  	    name = cur.prop("name")
  	    if name != None:
  		name = string.replace(name, "'", " ")
  		name = string.strip(name)
  	        addWord(name, file, symbol, 7)
          cur = cur.next
      if info == None:
          print "Function %s description has no <info>" % (symbol)
  	addFunction(symbol, file, "")
      else:
          info = string.replace(info, "'", " ")
  	info = string.strip(info)
  	addFunction(symbol, file, info)
          addString(info, file, symbol, 5)
  
      l = splitIdentifier(symbol)
      for word in l:
  	addWord(word, file, symbol, 10)
  
      return 1
  
  def analyzeAPISymbols(top):
      count = 0
      cur = top.children
          
      while cur != None:
          if cur.type == 'text':
  	    cur = cur.next
  	    continue
  	if cur.name == "macro":
  	    count = count + analyzeAPIMacro(cur)
  	elif cur.name == "function":
  	    count = count + analyzeAPIFunction(cur)
  	elif cur.name == "const":
  	    count = count + analyzeAPIConst(cur)
  	elif cur.name == "typedef":
  	    count = count + analyzeAPIType(cur)
  	elif cur.name == "struct":
  	    count = count + analyzeAPIStruct(cur)
  	elif cur.name == "enum":
  	    count = count + analyzeAPIEnum(cur)
  	elif cur.name == "functype":
  	    count = count + analyzeAPIFunctype(cur)
  	else:
  	    print "unexpected element %s in API doc <files>" % (cur.name)
          cur = cur.next
      return count
  
  def analyzeAPI(doc):
      count = 0
      if doc == None:
          return -1
      root = doc.getRootElement()
      if root.name != "api":
          print "Unexpected root name"
          return -1
      cur = root.children
      while cur != None:
          if cur.type == 'text':
  	    cur = cur.next
  	    continue
  	if cur.name == "files":
  	    pass
  #	    count = count + analyzeAPIFiles(cur)
  	elif cur.name == "symbols":
  	    count = count + analyzeAPISymbols(cur)
  	else:
  	    print "unexpected element %s in API doc" % (cur.name)
          cur = cur.next
      return count
  
  #########################################################################
  #									#
  #                  Web pages parsing and analysis			#
  #									#
  #########################################################################
  
  import glob
  
  def analyzeHTMLText(doc, resource, p, section, id):
      words = 0
      try:
  	content = p.content
  	words = words + addStringHTML(content, resource, id, section, 5)
      except:
          return -1
      return words
  
  def analyzeHTMLPara(doc, resource, p, section, id):
      words = 0
      try:
  	content = p.content
  	words = words + addStringHTML(content, resource, id, section, 5)
      except:
          return -1
      return words
  
  def analyzeHTMLPre(doc, resource, p, section, id):
      words = 0
      try:
  	content = p.content
  	words = words + addStringHTML(content, resource, id, section, 5)
      except:
          return -1
      return words
  
  def analyzeHTML(doc, resource, p, section, id):
      words = 0
      try:
  	content = p.content
  	words = words + addStringHTML(content, resource, id, section, 5)
      except:
          return -1
      return words
  
  def analyzeHTML(doc, resource):
      para = 0;
      ctxt = doc.xpathNewContext()
      try:
  	res = ctxt.xpathEval("//head/title")
  	title = res[0].content
      except:
          title = "Page %s" % (resource)
      addPage(resource, title)
      try:
  	items = ctxt.xpathEval("//h1 | //h2 | //h3 | //text()")
  	section = title
  	id = ""
  	for item in items:
  	    if item.name == 'h1' or item.name == 'h2' or item.name == 'h3':
  	        section = item.content
  		if item.prop("id"):
  		    id = item.prop("id")
  		elif item.prop("name"):
  		    id = item.prop("name")
  	    elif item.type == 'text':
  	        analyzeHTMLText(doc, resource, item, section, id)
  		para = para + 1
  	    elif item.name == 'p':
  	        analyzeHTMLPara(doc, resource, item, section, id)
  		para = para + 1
  	    elif item.name == 'pre':
  	        analyzeHTMLPre(doc, resource, item, section, id)
  		para = para + 1
  	    else:
  	        print "Page %s, unexpected %s element" % (resource, item.name)
      except:
          print "Page %s: problem analyzing" % (resource)
  	print sys.exc_type, sys.exc_value
  
      return para
  
  def analyzeHTMLPages():
      ret = 0
      HTMLfiles = glob.glob("*.html") + glob.glob("tutorial/*.html")
      for html in HTMLfiles:
  	if html[0:3] == "API":
  	    continue
  	if html == "xml.html":
  	    continue
  	try:
  	    doc = libxml2.parseFile(html)
  	except:
  	    doc = libxml2.htmlParseFile(html, None)
  	try:
  	    res = analyzeHTML(doc, html)
  	    print "Parsed %s : %d paragraphs" % (html, res)
  	    ret = ret + 1
  	except:
  	    print "could not parse %s" % (html)
      return ret
  
  #########################################################################
  #									#
  #                  Mail archives parsing and analysis			#
  #									#
  #########################################################################
  
  import time
  
  def getXMLDateArchive(t = None):
      if t == None:
  	t = time.time()
      T = time.gmtime(t)
      month = time.strftime("%B", T)
      year = T[0]
      url = "http://mail.gnome.org/archives/xml/%d-%s/date.html" % (year, month)
      return url
  
  def scanXMLMsgArchive(url, title, force = 0):
      if url == None or title == None:
          return 0
  
      ID = checkXMLMsgArchive(url)
      if force == 0 and ID != -1:
          return 0
  
      if ID == -1:
  	ID = addXMLMsgArchive(url, title)
  	if ID == -1:
  	    return 0
  
      try:
          print "Loading %s" % (url)
          doc = libxml2.htmlParseFile(url, None);
      except:
          doc = None
      if doc == None:
          print "Failed to parse %s" % (url)
  	return 0
  
      addStringArchive(title, ID, 20)
      ctxt = doc.xpathNewContext()
      texts = ctxt.xpathEval("//pre//text()")
      for text in texts:
          addStringArchive(text.content, ID, 5)
  
      return 1
  
  def scanXMLDateArchive(t = None, force = 0):
      global wordsDictArchive
  
      wordsDictArchive = {}
  
      url = getXMLDateArchive(t)
      print "loading %s" % (url)
      try:
  	doc = libxml2.htmlParseFile(url, None);
      except:
          doc = None
      if doc == None:
          print "Failed to parse %s" % (url)
  	return -1
      ctxt = doc.xpathNewContext()
      anchors = ctxt.xpathEval("//a[@href]")
      links = 0
      newmsg = 0
      for anchor in anchors:
  	href = anchor.prop("href")
  	if href == None or href[0:3] != "msg":
  	    continue
          try:
  	    links = links + 1
  
  	    msg = libxml2.buildURI(href, url)
  	    title = anchor.content
  	    if title != None and title[0:4] == 'Re: ':
  	        title = title[4:]
  	    if title != None and title[0:6] == '[xml] ':
  	        title = title[6:]
  	    newmsg = newmsg + scanXMLMsgArchive(msg, title, force)
  
  	except:
  	    pass
  
      return newmsg
      
  
  #########################################################################
  #									#
  #          Main code: open the DB, the API XML and analyze it		#
  #									#
  #########################################################################
  def analyzeArchives(t = None, force = 0):
      global wordsDictArchive
  
      ret = scanXMLDateArchive(t, force)
      print "Indexed %d words in %d archive pages" % (len(wordsDictArchive), ret)
  
      i = 0
      skipped = 0
      for word in wordsDictArchive.keys():
  	refs = wordsDictArchive[word]
  	if refs  == None:
  	    skipped = skipped + 1
  	    continue;
  	for id in refs.keys():
  	    relevance = refs[id]
  	    updateWordArchive(word, id, relevance)
  	    i = i + 1
  
      print "Found %d associations in HTML pages" % (i)
  
  def analyzeHTMLTop():
      global wordsDictHTML
  
      ret = analyzeHTMLPages()
      print "Indexed %d words in %d HTML pages" % (len(wordsDictHTML), ret)
  
      i = 0
      skipped = 0
      for word in wordsDictHTML.keys():
  	refs = wordsDictHTML[word]
  	if refs  == None:
  	    skipped = skipped + 1
  	    continue;
  	for resource in refs.keys():
  	    (relevance, id, section) = refs[resource]
  	    updateWordHTML(word, resource, section, id, relevance)
  	    i = i + 1
  
      print "Found %d associations in HTML pages" % (i)
  
  def analyzeAPITop():
      global wordsDict
      global API
  
      try:
  	doc = loadAPI(API)
  	ret = analyzeAPI(doc)
  	print "Analyzed %d blocs" % (ret)
  	doc.freeDoc()
      except:
  	print "Failed to parse and analyze %s" % (API)
  	print sys.exc_type, sys.exc_value
  	sys.exit(1)
  
      print "Indexed %d words" % (len(wordsDict))
      i = 0
      skipped = 0
      for word in wordsDict.keys():
  	refs = wordsDict[word]
  	if refs  == None:
  	    skipped = skipped + 1
  	    continue;
  	for (module, symbol) in refs.keys():
  	    updateWord(word, symbol, refs[(module, symbol)])
  	    i = i + 1
  
      print "Found %d associations, skipped %d words" % (i, skipped)
  
  def usage():
      print "Usage index.py [--force] [--archive]  [--archive-year year] [--archive-month month] [--API] [--docs]"
      sys.exit(1)
  
  def main():
      try:
  	openMySQL()
      except:
  	print "Failed to open the database"
  	print sys.exc_type, sys.exc_value
  	sys.exit(1)
  
      args = sys.argv[1:]
      force = 0
      if args:
          i = 0
  	while i < len(args):
  	    if args[i] == '--force':
  	        force = 1
  	    elif args[i] == '--archive':
  	        analyzeArchives(None, force)
  	    elif args[i] == '--archive-year':
  	        i = i + 1;
  		year = args[i]
  		months = ["January" , "February", "March", "April", "May",
  			  "June", "July", "August", "September", "October",
  			  "November", "December"];
  	        for month in months:
  		    try:
  		        str = "%s-%s" % (year, month)
  			T = time.strptime(str, "%Y-%B")
  			t = time.mktime(T) + 3600 * 24 * 10;
  			analyzeArchives(t, force)
  		    except:
  			print "Failed to index month archive:"
  			print sys.exc_type, sys.exc_value
  	    elif args[i] == '--archive-month':
  	        i = i + 1;
  		month = args[i]
  		try:
  		    T = time.strptime(month, "%Y-%B")
  		    t = time.mktime(T) + 3600 * 24 * 10;
  		    analyzeArchives(t, force)
  		except:
  		    print "Failed to index month archive:"
  		    print sys.exc_type, sys.exc_value
  	    elif args[i] == '--API':
  	        analyzeAPITop()
  	    elif args[i] == '--docs':
  	        analyzeHTMLTop()
  	    else:
  	        usage()
  	    i = i + 1
      else:
          usage()
  
  if __name__ == "__main__":
      main()
  

Download