[mentors-ops] r42 - doc importer/python website/cgi-bin/Mentors

chaas at mentors.debian.net chaas at mentors.debian.net
Fri Dec 30 12:14:34 CET 2005


Author: chaas
Date: 2005-12-30 12:14:33 +0100 (Fri, 30 Dec 2005)
New Revision: 42

Added:
   doc/dupload.conf
   doc/example-dupload-session
   doc/proftpd.conf
Modified:
   doc/TODO
   doc/concept.txt
   doc/mentors-database.txt
   importer/python/importy.py
   website/cgi-bin/Mentors/Database.py
   website/cgi-bin/Mentors/Importer.py
Log:
The works over christmas...

Improved debugging. Fixed bugs in the importer.

Added a few things to the concepts and todo list.


Modified: doc/TODO
===================================================================
--- doc/TODO	2005-12-08 11:52:59 UTC (rev 41)
+++ doc/TODO	2005-12-30 11:14:33 UTC (rev 42)
@@ -25,8 +25,7 @@
 - show the status of the current upload queue
 - show the bugs from the BTS in a package page (ldap://bts2ldap.debian.net:10101)
 - help the user to file an ITP (create a template)
-- send someone (e.g. email address of a sponsor) information about a
-  package on mentors
+- prepare (or seven send on behalf) an RFS for debian-mentors at l.d.o
 
 Importer:
 - ChrisH is currently rewriting his importer in Python.
@@ -38,6 +37,8 @@
 - automatic lintian/linda checks
 - check for native packages
 - only accept uploads for registered users ("Maintainer:" field)
+- check if the author has changed (debian/changelog) to tell the user
+  if an ITA should be filed
 
 Database:
 - Neil should document his database layout in doc/database.txt

Modified: doc/concept.txt
===================================================================
--- doc/concept.txt	2005-12-08 11:52:59 UTC (rev 41)
+++ doc/concept.txt	2005-12-30 11:14:33 UTC (rev 42)
@@ -3,5 +3,7 @@
 
 - There is always only one version/revision of a package stored.
   The importer has to remove older versions during the import.
-- Only allow newer revisions in (arguable!)
+- (Only allow newer revisions in. Probably not a good idea since
+  sponsorees upload packages multiple times with the same revision
+  until it's clean to be uploaded to Debian.)
 

Added: doc/dupload.conf
===================================================================
--- doc/dupload.conf	2005-12-08 11:52:59 UTC (rev 41)
+++ doc/dupload.conf	2005-12-30 11:14:33 UTC (rev 42)
@@ -0,0 +1,7 @@
+$cfg{'mentors'} =
+{
+        fqdn => 'mentors.debian.net',
+        incoming => '/',
+        login => 'email at christoph-haas.de',
+        dinstall_runs => 1,
+};

Added: doc/example-dupload-session
===================================================================
--- doc/example-dupload-session	2005-12-08 11:52:59 UTC (rev 41)
+++ doc/example-dupload-session	2005-12-30 11:14:33 UTC (rev 42)
@@ -0,0 +1,20 @@
+$> dupload -t mentors comix_1.3.1-1_i386.changes
+dupload note: no announcement will be sent.
+Checking signatures before upload......signatures are ok
+Uploading (ftp) to mentors.debian.net:/
+[ job comix_1.3.1-1_i386 from comix_1.3.1-1_i386.changes
+ comix_1.3.1-1.diff.gz, md5sum ok
+ comix_1.3.1-1_i386.deb, md5sum ok
+ comix_1.3.1.orig.tar.gz, md5sum ok
+ comix_1.3.1-1.dsc, md5sum ok
+ comix_1.3.1-1_i386.changes ok ]
+user at domain@mentors.debian.net's ftp account password:
+Uploading (ftp) to mentors2 (mentors.debian.net)
++ FTP passive mode selected
+[ Uploading job comix_1.3.1-1_i386
+ comix_1.3.1-1.diff.gz 2.6 kB, ok (0 s, 2.61 kB/s)
+ comix_1.3.1-1_i386.deb 29.5 kB, ok (0 s, 29.46 kB/s)
+ comix_1.3.1.orig.tar.gz 33.3 kB, ok (0 s, 33.30 kB/s)
+ comix_1.3.1-1.dsc 0.6 kB, ok (0 s, 0.56 kB/s)
+ comix_1.3.1-1_i386.changes 1.9 kB, ok (0 s, 1.89 kB/s) ]
+

Modified: doc/mentors-database.txt
===================================================================
--- doc/mentors-database.txt	2005-12-08 11:52:59 UTC (rev 41)
+++ doc/mentors-database.txt	2005-12-30 11:14:33 UTC (rev 42)
@@ -1,46 +1,61 @@
-WARNING: This is the database layout of m.d.n. We need to see how we
-	can integrate it into s.d.n.
-
 ======================================================================
 === DATABASE LAYOUT
 ======================================================================
 
+-- 
+-- Tabellenstruktur für Tabelle `packages`
+-- 
+
 CREATE TABLE `packages` (
-  `id` int(11) NOT NULL auto_increment,
   `user_id` int(11) NOT NULL default '0',
-  `maintainer` varchar(100) NOT NULL default '',
   `package` varchar(50) NOT NULL default '',
   `path` varchar(200) NOT NULL default '',
-  `description` varchar(255) NOT NULL default '',
+  `description` text NOT NULL,
   `version` varchar(50) NOT NULL default '',
   `section` varchar(50) NOT NULL default '',
-  `age` double NOT NULL default '0',
-  PRIMARY KEY  (`id`)
-) TYPE=MyISAM AUTO_INCREMENT=3377 ;
+  `priority` varchar(50) NOT NULL default '',
+  `import_timestamp` timestamp(14) NOT NULL,
+  `lintian_report` text NOT NULL,
+  `lintian_errors` int(11) NOT NULL default '0',
+  `lintian_warnings` int(11) NOT NULL default '0',
+  `closes_bugs` text NOT NULL,
+  UNIQUE KEY `package` (`package`)
+) TYPE=MyISAM COMMENT='Test table for the importer';
 
+-- --------------------------------------------------------
+
+-- 
+-- Tabellenstruktur für Tabelle `sessions`
+-- 
+
 CREATE TABLE `sessions` (
   `sessionid` varchar(40) NOT NULL default '',
   `time` timestamp(14) NOT NULL,
   `ipaddress` varchar(15) NOT NULL default '',
   `username` varchar(80) default NULL,
-  UNIQUE KEY `sessionid` (`sessionid`)
+  UNIQUE KEY `sessionid` (`sessionid`),
+  UNIQUE KEY `username` (`username`)
 ) TYPE=MyISAM COMMENT='Session information corresponding to HTTP cookies being sent';
 
+-- --------------------------------------------------------
+
+-- 
+-- Tabellenstruktur für Tabelle `users`
+-- 
+
 CREATE TABLE `users` (
   `id` int(11) NOT NULL auto_increment,
   `realname` varchar(50) NOT NULL default '',
   `email` varchar(50) NOT NULL default '',
   `country` varchar(50) NOT NULL default '',
   `ircnick` varchar(30) NOT NULL default '',
-  `ssh2key` mediumtext NOT NULL,
   `gpgkey` mediumtext NOT NULL,
-  `comment` varchar(240) NOT NULL default '',
   `status` enum('new','active') NOT NULL default 'new',
   `password` varchar(15) NOT NULL default '',
   `hash` varchar(50) default NULL,
   PRIMARY KEY  (`id`),
   UNIQUE KEY `email` (`email`)
-) TYPE=MyISAM AUTO_INCREMENT=400 ;
+) TYPE=MyISAM;
 
 ======================================================================
 === MEANING OF DATABASE ENTRIES
@@ -49,13 +64,18 @@
 packages: (database of packages uploaded and currently in the repository)
 	- id = auto-incrementing index of database entries
 	- user_id = id of the uploader from the `users` table
-	- maintainer = email of uploader (deprecated field)
 	- package = name of the source package
 	- path = relative path to the package directory on the web server
+	  (e.g. pool/main/c/cream)
 	- description = one-line description from control file
 	- version = package revision
 	- section = package section
-	- age = timestamp - UNIX ticks (should become a timestamp)
+	- priority = package priority (optional, ...)
+	- import_timestamp = time the package was imported
+	- lintian_report = texttual output from lintian
+	- lintian_errors = number of lintian errors (-1 = not lintian checked)
+	- lintian_warnings = number of lintian warnings (-1 = not lintian checked)
+	- closes_bugs = numbers of bugs that this package would close
 
 session: (table of cookie-based sessions to authenticate web users)
 	- sessionid = random session id (MD5 hash of a random value)
@@ -69,9 +89,7 @@
 	- email = email address of the user
 	- country = name of the country the user comes from
 	- ircnick = IRC nickname on freenode.net (optional)
-	- ssh2key = ssh2 public key for logging in (deprecated)
 	- gpgkey = gpg public key the user signs his packages with
-	- comment = comment to the mentors.debian.net team (deprecated)
 	- status = 'new'=signed up / 'active'=verified and active
 	- password = web/ftp password for logging in and uploading through FTP
 	- hash = one-time hash to verify the user when signing up for a new account

Added: doc/proftpd.conf
===================================================================
--- doc/proftpd.conf	2005-12-08 11:52:59 UTC (rev 41)
+++ doc/proftpd.conf	2005-12-30 11:14:33 UTC (rev 42)
@@ -0,0 +1,60 @@
+ServerName      "mentors.debian.net"
+ServerType      standalone
+DeferWelcome      off
+
+MultilineRFC2228    on
+DefaultServer     on
+ShowSymlinks      on
+
+TimeoutNoTransfer   600
+TimeoutStalled      600
+TimeoutIdle     1200
+
+DisplayLogin                    welcome.msg
+DisplayFirstChdir               .message
+ListOptions                 "-l"
+
+IdentLookups off
+
+DenyFilter      \*.*/
+
+# Port 21 is the standard FTP port.
+Port        21
+
+# To prevent DoS attacks, set the maximum number of child processes
+# to 30.  If you need to allow more than 30 concurrent connections
+# at once, simply increase this value.  Note that this ONLY works
+# in standalone mode, in inetd mode you should use an inetd server
+# that allows you to limit maximum number of processes per service
+# (such as xinetd)
+MaxInstances      30
+
+# Set the user and group that the server normally runs at.
+User        incoming
+Group       nogroup
+
+# Umask 022 is a good standard umask to prevent new files and dirs
+# (second parm) from being group and world writable.
+Umask       022  022
+# Normally, we want files to be overwriteable.
+AllowOverwrite      on
+
+DefaultRoot ~
+
+SQLConnectInfo mentors at localhost readme QM7OJ4zomXSz
+SQLUserInfo users email password NULL NULL '/home/incoming' NULL
+SQLUserWhereClause "password != ''"
+SQLAuthenticate users*
+SQLAuthTypes Plaintext
+RequireValidShell off
+SQLDefaultUID 1111
+SQLDefaultGID 100
+
+UseReverseDNS off
+
+ <Limit READ WRITE>
+   DenyAll
+ </Limit>
+ <Limit STOR>
+  AllowAll
+ </Limit>

Modified: importer/python/importy.py
===================================================================
--- importer/python/importy.py	2005-12-08 11:52:59 UTC (rev 41)
+++ importer/python/importy.py	2005-12-30 11:14:33 UTC (rev 42)
@@ -13,7 +13,7 @@
 # - check if all the binaries (gpg, lintian, dpkg) are there
 # - the orig tarball mustn't change for the same upstream release
 
-import sys, re
+import sys, re, string, os
 
 sys.path.append('/etc/mentors')
 import Config # global config from /etc/mentors
@@ -61,14 +61,16 @@
 def main():
    dbcursor = database.cursor()
 
+   os.chdir(options.incomingdir)
+
    # Glob through all the *.changes files in the incoming directory
    for changes_filename in I.listControlFiles('changes'):
       # Parse the changes file
       ###for dsc_filename in I.listControlFiles('dsc'):
-      I.debug("Changes found: %s" % changes_filename)
+      I.debug("parse-changes","Parsing Changes file: %s" % changes_filename)
       changes_fields,signer_changes = I.parseControlFile(changes_filename)
 
-      I.debug("The changes file was signed by: %s" % signer_changes)
+      I.debug("parse-changes","The changes file was signed by: %s" % signer_changes)
 
       # Get the pointer to the dsc file from the list of "Files"
       # in the changes file
@@ -85,7 +87,7 @@
       pkg_priority = dsc_line[3]
       dsc_filename = dsc_line[4]
 
-      I.debug("Changes file says the DSC file is: %s" % dsc_filename)
+      I.debug("parse-changes","Changes file says the DSC file is: %s" % dsc_filename)
 
       # Parse the *.dsc file
       dsc_fields,signer_dsc = I.parseControlFile(dsc_filename)
@@ -93,31 +95,31 @@
       # Calculate the upstream version
       match = re.search(r'^(.*)-(\d+)$', dsc_fields['Version'])
       upstream_version, revision = match.groups()
-      I.debug("Upstream version is: %s" % upstream_version )
-      I.debug("Revision is: %s" % revision )
+      I.debug("parse-changes", "Upstream version is: %s" % upstream_version )
+      I.debug("parse-changes","Revision is: %s" % revision )
 
       # Create a list of files inside the package (Parse the "Files" entry)
       uploaded_files = [line.split()[2] for line in dsc_fields['Files'] ]
 
       # Go through the files listed in the "Files" entry and check each
       for line in dsc_fields["Files"]:
-         I.debug("File entry: %s" % line)
+         I.debug("parse-changes","File entry: %s" % line)
          checksum, size, filename = line.split()
-         I.debug("File data: checksum=%s, size=%s, filename=%s" % \
+         I.debug("parse-changes","File data: checksum=%s, size=%s, filename=%s" % \
             (checksum, size, filename) )
 
          # Check if the file mentioned in the control file is correctly there
          return_code, error_message = \
             I.checkFile(checksum=checksum, size=int(size), filename=filename)
 
-         I.debug("Return code from checkFile: %s (%s)" % (return_code,
+         I.debug("parse-changes","Return code from checkFile: %s (%s)" % (return_code,
             error_message) )
 
          if return_code == I.OK:
-            I.debug("File check passed for %s" % filename)
+            I.debug("parse-changes","File check passed for %s" % filename)
             #uploaded_files.append(filename)
          elif return_code == I.WARN:
-            I.debug("File check failed for %s" % filename)
+            I.debug("parse-changes","File check failed for %s" % filename)
             # TODO: write this warning into the database or something
          else: # return_code == I.ERR:
             I.sendMail(template="uploadfailed",
@@ -125,66 +127,122 @@
                errormessage=error_message,
                packagename=dsc_fields["Source"])
             #I.fatal(error_message)
-            I.debug("Import failed.")
+            I.debug("parse-changes!","Import failed.")
             #I.removeFiles()
             # TODO: remove files
             return
 
-      I.debug("Successfully uploaded files: %s" % uploaded_files)
+      I.debug("parse-changes","Successfully uploaded files: %s" % uploaded_files)
 
       # Check for native package (no orig file)
       designated_orig_tarball = "%s_%s.orig.tar.gz" % (dsc_fields['Source'], upstream_version)
-      I.debug("Designated orig tarball is: %s" % designated_orig_tarball )
+      I.debug("parse-changes","Designated orig tarball is: %s" % designated_orig_tarball )
       if designated_orig_tarball not in uploaded_files:
-         I.debug("Orig tarball not found in uploaded files. Native package.")
+         I.debug("parse-changes","Orig tarball not found in uploaded files. Native package.")
          print "* Warning: Native package"
          # TODO: add this warning to the database
       else:
-         I.debug("Orig tarball found. Non-native package. Ok.")
+         I.debug("parse-changes","Orig tarball found. Non-native package. Ok.")
 
    # Package integrity is verified now
-   I.debug("-------- Package integrity checked ---------")
+   I.debug("main","--- Package integrity checked ---")
 
    #-- Find out if we know the maintainer
-   I.debug("Checking if the maintainer is a registered user")
+   I.debug("check-pkg","Checking if the maintainer is a registered user")
    sql = """select id from %s where concat(realname,' <',email,'>')=%%s""" % \
       Config.DatabaseTableUsers
-   I.debug("SQL select query for maintainer: %s" % sql)
-   I.debug("SQL parameters for this query: %s" % dsc_fields['Maintainer'])
+   I.debug("check-pkg","SQL select query for maintainer: %s" % sql)
+   I.debug("check-pkg","SQL parameters for this query: %s" % dsc_fields['Maintainer'])
 
-   I.debug("Executing select query.")
+   I.debug("check-pkg","Executing select query.")
    dbcursor.execute(sql, dsc_fields['Maintainer'])
    maintainer_row = dbcursor.fetchone()
    if maintainer_row is None:
-      I.debug("* Maintainer %s is unknown in database. Not yet registered?" %
+      I.debug("check-pkg","* Maintainer %s is unknown in database. Not yet registered?" %
          dsc_fields['Maintainer'])
       I.sendMail(template="maintainerunknown",
          to=dsc_fields["Maintainer"],
          packagename=dsc_fields["Source"])
-   I.debug("Maintainer found in database. Has ID: %s" % maintainer_row['id'])
+   I.debug("check-pkg","Maintainer found in database. Has ID: %s" % maintainer_row['id'])
 
    #-- Lintian check
-   returncode, lintian_report, stderr = I.run("/usr/bin/lintian -i %s" %
-      changes_filename)
+   if Config.ImporterDoLintianChecks:
+      lintian_report = ["Package was not lintian checked."]
+      lintian_errors = -1
+      lintian_warnings = -1
+   else:
+      returncode, lintian_report, stderr = I.run("/usr/bin/lintian -i %s" %
+         changes_filename)
 
-   if returncode not in (0,1):
-      I.fatal("Lintian returned with code %s" % returncode)
+      if returncode not in (0,1):
+         I.fatal("Lintian returned with code %s" % returncode)
 
-   I.debug("Lintian return code: %s" % returncode)
-   I.debug("Lintian report: %s" % lintian_report)
+      I.debug("check-pkg","Lintian return code: %s" % returncode)
+      I.debug("check-pkg","Lintian report: %s" % lintian_report)
 
-   # Lines in lintian reports starting with "W:" mark warnings.
-   # Same with "E:" for errors.
-   lintian_warnings = len( 
-      [warning for warning in lintian_report if warning.startswith("W: ")] )
-   lintian_errors = len( 
-      [warning for warning in lintian_report if warning.startswith("E: ")] )
+      # Lines in lintian reports starting with "W:" mark warnings.
+      # Same with "E:" for errors.
+      lintian_warnings = len( 
+         [warning for warning in lintian_report if warning.startswith("W: ")] )
+      lintian_errors = len( 
+         [warning for warning in lintian_report if warning.startswith("E: ")] )
 
+   # Split the section (editor/non-free)
+   if '/' in pkg_section:
+      pkg_supersection, pkg_subsection = string.split(pkg_section, '/')
+   else:
+      pkg_supersection = 'main'
+      pkg_subsection = pkg_section
+
+   I.debug("check-pkg","Supersection is: %s" % pkg_supersection)
+   I.debug("check-pkg","Subsection is: %s" % pkg_subsection)
+
+   I.debug("move-pkg","Moving files into repository")
+
+   # Compute the parent directory of where the package will be moved to
+   # /home/apt-get/pool/main/c
+   destination_parentdir = "%s/pool/%s/%s" % (
+      Config.ImporterRepositoryPath,
+      pkg_supersection,
+      dsc_fields['Source'][0] # first char of the package name
+      )
+
+   # Compute the destination directory of this package
+   # /home/apt-get/pool/main/c/cream/
+   destination_dir = "%s/%s" % (
+      destination_parentdir,
+      dsc_fields['Source'], # name of the source package
+      )
+
+   # Compute the URL where the package can be found later
+   pkg_url = "pool/%s/%s/%s" % (
+      pkg_supersection,
+      dsc_fields['Source'][0], # first char of the package name
+      dsc_fields['Source'], # name of the source package
+      )
+
+   I.debug("move-pkg","Path to put move this package to: %s" % destination_dir)
+
+   # Create the repository directory tree if needed
+   if not os.path.exists(destination_parentdir):
+      I.create_repository_directory_tree()
+
+   # Create the package's directory if needed
+   if not os.path.exists(destination_dir):
+      os.mkdir(destination_dir)
+
    # Move files into place
-   #...
+   if options.simulate:
+      I.debug("move-pkg","Simulation mode - not moving files into %s" % destination_path)
+   else:
+      # Upload .dsc file + all files mentioned in the .dsc file
+      for file in ([dsc_filename] + uploaded_files):
+         newfile = os.path.join(destination_dir, file)
+         I.debug("move-pkg","Moving %s to %s" % (file, newfile))
+         os.rename(file, newfile)
 
    # Add information about this package to the database
-   I.debug("Inserting package information into the database")
+   I.debug("insert-db","Inserting package information into the database")
    sql = """replace into %s set user_id=%%s,package=%%s,path=%%s,
       description=%%s,version=%%s,section=%%s,priority=%%s,
       lintian_report=%%s,lintian_warnings=%%s,lintian_errors=%%s,
@@ -193,10 +251,10 @@
    sqlparams = (
       maintainer_row['id'], # user id
       dsc_fields['Source'], # source package name
-      "FIXME: path", # path in the repository
+      pkg_url, # URL in the repository where the package is stored
       "\n".join(changes_fields['Description']), # description
       dsc_fields['Version'], # version
-      pkg_section, # section
+      pkg_section, # subsection
       pkg_priority, # priority
       "\n".join(lintian_report),
       lintian_warnings, # number of lintian warnings
@@ -204,16 +262,15 @@
       changes_fields['Closes'], # closed bugs
       )
 
-   I.debug("SQL replace query for new package: %s" % sql)
-   I.debug("SQL parameters for this query: %s" % str(sqlparams))
+   I.debug("insert-db","SQL replace query for new package: %s" % sql)
+   I.debug("insert-db","SQL parameters for this query: %s" % str(sqlparams))
 
    if options.simulate:
-      I.debug("Simulation mode - not executing query")
+      I.debug("insert-db","Simulation mode - not executing query")
    else:
-      I.debug("Executing replace query.")
+      I.debug("insert-db","Executing replace query.")
       dbcursor.execute(sql, sqlparams)
 
-   #I.debug("Moving files into repository")
 
 ###########################################################################
 
@@ -222,10 +279,12 @@
    main()
 except:
    import traceback
-   I.debug("The main() loop has thrown up. Traceback:\n%s" %
-      traceback.format_exc())
+   print traceback.format_exc()
+   #I.debug("main","The main() loop has thrown up. Traceback:\n%s" %
+   #   traceback.format_exc())
    # TODO: write this traceback to a log file
    print "Aborting..."
+   sys.exit(10)
 
 # vim:set shiftwidth=3 expandtab textwidth=78 smarttab autoindent:
 

Modified: website/cgi-bin/Mentors/Database.py
===================================================================
--- website/cgi-bin/Mentors/Database.py	2005-12-08 11:52:59 UTC (rev 41)
+++ website/cgi-bin/Mentors/Database.py	2005-12-30 11:14:33 UTC (rev 42)
@@ -3,12 +3,17 @@
 """
 
 import MySQLdb
+import Config
 
 class Database:
    def __init__(self):
       # Open the database
       try:
-         self.dbhandle = MySQLdb.connect(host="localhost", user="torfdev", passwd="mentorsrox", db="mentors" )   
+         self.dbhandle = MySQLdb.connect(
+            host=Config.DatabaseHost,
+            user=Config.DatabaseUser,
+            passwd=Config.DatabasePassword,
+            db=Config.DatabaseName )   
       except MySQLdb.OperationalError, message:
          print "Database Error %d:\n%s" % (message[ 0 ], message[ 1 ] )
 

Modified: website/cgi-bin/Mentors/Importer.py
===================================================================
--- website/cgi-bin/Mentors/Importer.py	2005-12-08 11:52:59 UTC (rev 41)
+++ website/cgi-bin/Mentors/Importer.py	2005-12-30 11:14:33 UTC (rev 42)
@@ -14,6 +14,8 @@
 import re
 import os
 import subprocess
+import Config
+import string
 
 # Utility class
 class Util:
@@ -32,9 +34,9 @@
          self.fatal("uploadDir must be specified")
 
    # print debug
-   def debug(self,string):
+   def debug(self, section, string):
       if self.debugEnabled == True:
-         print "DEBUG:", string
+         print "DEBUG[%s]: %s" % (section, string)
 
    # Deal with a fatal error that has occured
    def fatal(self,string):
@@ -44,13 +46,13 @@
 
    # Get a list of *.changes/*.dsc files
    def listControlFiles(self, suffix):
-      self.debug("Looking for *.%s files in %s" % (suffix, self.uploadDir,) )
+      self.debug("listControlFiles","Looking for *.%s files in %s" % (suffix, self.uploadDir,) )
       return glob.glob("%s/*.%s" % (self.uploadDir, suffix) )
       #return glob.glob(self.uploadDir + '/*.dsc')
 
    # Run an external command and return stdout and stderr
    def run(self,command):
-      self.debug("Executing: %s" % command)
+      self.debug("run-ext","Executing: %s" % command)
       run = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
       # Wait for the process to return
@@ -66,14 +68,18 @@
 
    # Read a control file (.dsc or .changes) and parse it
    def parseControlFile(self,dsc_filename):
-      self.debug("Parsing file: %s" % dsc_filename)
-      self.debug("Stripping off PGP header")
+      self.debug("parseControlFile","Parsing file: %s" % dsc_filename)
+      self.debug("parseControlFile","Stripping off PGP header")
       returncode,dsc_lines,stderr = self.run(self.gpg + " --decrypt " + dsc_filename)
-      self.debug("PGP return code: %s" % returncode)
-      self.debug("PGP stderr: %s" % stderr)
+      self.debug("parseControlFile","PGP return code: %s" % returncode)
+      self.debug("parseControlFile","PGP stderr: %s" % stderr)
       if returncode in (1,2):
+         self.debug("parseControlFile!","PGP signature is invalid: " +
+            '\n'.join(stderr))
          self.fatal("Invalid PGP signature: " + '\n'.join(stderr))
       elif returncode > 2:
+         self.debug("parseControlFile!","PGP error: " +
+            '\n'.join(stderr))
          self.fatal("Serious trouble when calling GPG: " + '\n'.join(stderr))
 
       # The name and email address of the signer is sent to stderr when
@@ -92,22 +98,22 @@
       entries = {}
 
       for line in dsc_lines:
-         self.debug("Parsing line: |%s|" % line)
+         self.debug("parseControlFile","Parsing line: |%s|" % line)
 
          if line == "":
-            self.debug("Skipping empty line")
+            self.debug("parseControlFile","Skipping empty line")
             continue
 
          # Line contains a "Key: ..."
          if ":" in line:
-            self.debug("Single-line entry found")
+            self.debug("parseControlFile","Single-line entry found")
             match = re.search(r'^(.*?):(.*)', line)
             key, value = match.groups()
             #key,value = line.split(":")
             value = value.strip()
 
-            self.debug("Key: " + key)
-            self.debug("Value: " + value)
+            self.debug("parseControlFile","Key: " + key)
+            self.debug("parseControlFile","Value: " + value)
 
             if value == "":
                # Start of multi-line entry? ("Key:")
@@ -118,10 +124,10 @@
                entries[key] = value
          else:
             # Multi-line entry ("Key: \n ....\n ....")
-            self.debug("Multi-line entry found")
+            self.debug("parseControlFile","Multi-line entry found")
             entries[key].append(line[1:]) # strip first space char and add line
 
-      self.debug("Parsed entries: %s" % entries)
+      self.debug("parseControlFile","Parsed entries: %s" % entries)
       return entries,signer
 
    # Check if a file mentioned in the control file is correctly there
@@ -130,29 +136,29 @@
       abs_filename = os.path.join(self.uploadDir, filename)
 
       # Check if the file exists
-      self.debug("Checking if file %s really exists" % abs_filename)
+      self.debug("checkFile","Checking if file %s really exists" % abs_filename)
       if not os.path.isfile(abs_filename):
-         self.debug("* File not found")
+         self.debug("checkFile","* File not found")
          return (Util.ERROR, ("The file '%s' which is listed in your dsc file\n" +
             "was not found.") % filename )
 
       # Check the file size
-      self.debug("Checking if file %s really has size %s" % (filename,size))
+      self.debug("checkFile","Checking if file %s really has size %s" % (filename,size))
       real_size = os.path.getsize(abs_filename)
       if size != real_size:
-         self.debug("* File size mismatch.")
+         self.debug("checkFile","* File size mismatch.")
          error_message = "The file '%s' should be %d bytes large. But it\n" + \
             "rather appears to be %d bytes large."
          return Util.ERROR, error_message % (filename, size, real_size)
 
       # Check the MD5 checksum
-      self.debug("Checking if file %s has checksum %s" % (filename,checksum))
+      self.debug("checkFile","Checking if file %s has checksum %s" % (filename,checksum))
       if checksum != self.checkMD5(abs_filename):
          error_message = "The file '%s' does not have the correct checksum." \
             % (filename)
          return Util.ERROR, error_message
 
-      self.debug("File check o.k.")
+      self.debug("checkFile","File check o.k.")
       return Util.OK, None
 
    # Get the MD5 checksum of a file
@@ -166,13 +172,33 @@
    def sendMail(self, template, **variables):
       # Add the standard sender address to the dictionary
       variables["from"]=self.emailSender
-      self.debug("Sending mail from template '%s'" % template)
-      self.debug("Template variables '%s'" % variables)
+      self.debug("send-mail","Sending mail from template '%s'" % template)
+      self.debug("send-mail","Template variables '%s'" % variables)
       template_file = os.path.join(self.templateDir, template)
       file = open(template_file)
       mail = "".join(file.readlines()) % variables
-      self.debug("Sending mail:\n%s" % mail )
+      self.debug("send-mail","Sending mail:\n%s" % mail )
       #TODO: complete this :)
 
+   # Create the repository directory tree
+   def create_repository_directory_tree(self):
+      # Create a directory inside the repository directory
+      def makedir(*subdirs):
+         full_path = Config.ImporterRepositoryPath
+         if len(subdirs) > 0:
+            full_path = os.path.join(full_path, *subdirs)
+         if not os.path.isdir(full_path):
+            os.makedirs(full_path)
+
+      self.debug("create-repo","Creating repository tree")
+      makedir('dists')
+
+      for dir in ['main', 'contrib', 'non-free']:
+         for char in string.lowercase: # a-z
+            makedir('pool', dir, char)
+
+      for dir in ['main', 'contrib', 'non-free']:
+         makedir('dists', 'unstable', dir, 'source')
+
 # vim:set shiftwidth=3 expandtab textwidth=78 smarttab autoindent:
 




More information about the mentors-ops mailing list